You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2018/05/17 23:29:57 UTC
[incubator-pulsar] branch master updated: Added docker files and
script to build static client library and Python bindings on Linux (#1793)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 1a0c8dc Added docker files and script to build static client library and Python bindings on Linux (#1793)
1a0c8dc is described below
commit 1a0c8dcea29feb3d2cee71e4fcc3cab6b06e65bf
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Thu May 17 16:29:53 2018 -0700
Added docker files and script to build static client library and Python bindings on Linux (#1793)
* Added docker files and script to build static client library and Python bindings on Linux
* Added license headers
---
pulsar-client-cpp/docker/Dockerfile | 144 +++++++++++++++++++++
.../docker/build-client-lib-within-docker.sh | 34 +++++
pulsar-client-cpp/docker/build-client-lib.sh | 44 +++++++
.../docker/build-wheel-file-within-docker.sh | 35 +++++
pulsar-client-cpp/docker/build-wheels.sh | 48 +++++++
pulsar-client-cpp/docker/create-images.sh | 48 +++++++
pulsar-client-cpp/docker/push-images.sh | 51 ++++++++
7 files changed, 404 insertions(+)
diff --git a/pulsar-client-cpp/docker/Dockerfile b/pulsar-client-cpp/docker/Dockerfile
new file mode 100644
index 0000000..1df9545
--- /dev/null
+++ b/pulsar-client-cpp/docker/Dockerfile
@@ -0,0 +1,144 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+FROM quay.io/pypa/manylinux1_x86_64
+
+RUN yum install -y gtest-devel
+
+ARG PYTHON_VERSION
+ARG PYTHON_SPEC
+
+ENV PYTHON_VERSION=${PYTHON_VERSION}
+ENV PYTHON_SPEC=${PYTHON_SPEC}
+
+ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
+
+RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
+
+####################################
+# These dependencies can be found in Ubuntu but they're not compiled with -fPIC,
+# so they cannot be statically linked into a shared library
+####################################
+
+# ZLib
+RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz && \
+ tar xvfz zlib-1.2.11.tar.gz && \
+ cd zlib-1.2.11 && \
+ CFLAGS="-fPIC -O3" ./configure && \
+ make && make install && \
+ rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11
+
+# Compile OpenSSL
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_0_2l.tar.gz && \
+ tar xvfz OpenSSL_1_0_2l.tar.gz && \
+ cd openssl-OpenSSL_1_0_2l/ && \
+ ./Configure -fPIC no-shared linux-x86_64 && \
+ make && make install && \
+ rm -rf /OpenSSL_1_0_2l.tar.gz /openssl-OpenSSL_1_0_2l
+
+# Download and compile boost
+RUN curl -O -L https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz && \
+ tar xvfz boost_1_64_0.tar.gz && \
+ cd /boost_1_64_0 && \
+ ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
+ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
+ rm -rf /boost_1_64_0.tar.gz /boost_1_64_0
+
+# Download and copile protoubf
+RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \
+ tar xvfz protobuf-cpp-3.3.0.tar.gz && \
+ cd protobuf-3.3.0/ && \
+ CXXFLAGS=-fPIC ./configure && \
+ make && make install && ldconfig && \
+ rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0
+
+# Compile APR
+RUN curl -O -L http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \
+ tar xvfz apr-1.5.2.tar.gz && \
+ cd apr-1.5.2 && \
+ CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
+ make && make install && \
+ rm -rf /apr-1.5.2.tar.gz /apr-1.5.2
+
+# Compile APR-Util
+RUN curl -O -L http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \
+ tar xvfz apr-util-1.5.4.tar.gz && \
+ cd apr-util-1.5.4 && \
+ CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \
+ make && make install && \
+ rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4
+
+# Libtool
+RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \
+ tar xvfz libtool-2.4.6.tar.gz && \
+ cd libtool-2.4.6 && \
+ ./configure && \
+ make && make install && \
+ rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6
+
+# Compile log4cxx
+RUN git clone https://github.com/apache/logging-log4cxx.git && \
+ cd logging-log4cxx && \
+ ./autogen.sh && \
+ CXXFLAGS=-fPIC ./configure && \
+ make && make install && \
+ rm -rf /logging-log4cxx
+
+# Compile expat
+RUN curl -O -L https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \
+ tar xfvz R_2_2_0.tar.gz && \
+ cd libexpat-R_2_2_0/expat && \
+ ./buildconf.sh && \
+ CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
+ make && make installlib && \
+ rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0
+
+RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.8.2.tar.gz && \
+ tar xvfz v3.8.2.tar.gz && \
+ cd CMake-3.8.2 && \
+ ./configure && \
+ make && make install && \
+ rm -rf /v3.8.2.tar.gz /CMake-3.8.2
+
+# Compile JSON CPP
+RUN curl -O -L https://github.com/open-source-parsers/jsoncpp/archive/1.8.0.tar.gz && \
+ tar xvfz 1.8.0.tar.gz && \
+ cd jsoncpp-1.8.0 && \
+ cmake . -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \
+ make && make install && \
+ rm -rf /1.8.0.tar.gz /jsoncpp-1.8.0
+
+# LibCurl
+RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_54_0/curl-7.54.0.tar.gz && \
+ tar xvfz curl-7.54.0.tar.gz && \
+ cd curl-7.54.0 && \
+ CFLAGS=-fPIC ./configure && \
+ make && make install && \
+ rm -rf /curl-7.54.0.tar.gz /curl-7.54.0
+
+RUN pip install twine
+
+
+ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include
+ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
+ENV OPENSSL_ROOT_DIR /usr/local/ssl/
+
+COPY build-wheel-file-within-docker.sh /
+COPY build-client-lib-within-docker.sh /
diff --git a/pulsar-client-cpp/docker/build-client-lib-within-docker.sh b/pulsar-client-cpp/docker/build-client-lib-within-docker.sh
new file mode 100755
index 0000000..3a356cc
--- /dev/null
+++ b/pulsar-client-cpp/docker/build-client-lib-within-docker.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+set -e -x
+
+cd /pulsar/pulsar-client-cpp
+
+find . -name CMakeCache.txt | xargs rm
+find . -name CMakeFiles | xargs rm -rf
+rm lib/*.pb.*
+
+cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON \
+ -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERSION \
+ -DPYTHON_LIBRARY=/opt/python/$PYTHON_SPEC/lib \
+
+make pulsarShared pulsarStatic -j4
diff --git a/pulsar-client-cpp/docker/build-client-lib.sh b/pulsar-client-cpp/docker/build-client-lib.sh
new file mode 100755
index 0000000..61f2d66
--- /dev/null
+++ b/pulsar-client-cpp/docker/build-client-lib.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+set -e
+
+BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd $ROOT_DIR
+
+PYTHON_VERSIONS=(
+ '3.6 cp36-cp36m'
+)
+
+for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a PY <<< "$line"
+ PYTHON_VERSION=${PY[0]}
+ PYTHON_SPEC=${PY[1]}
+ echo "--------- Build Client library"
+
+ IMAGE_NAME=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+
+ echo "Using image: $IMAGE_NAME"
+
+ docker run -i -v $PWD:/pulsar $IMAGE_NAME /build-client-lib-within-docker.sh
+done
diff --git a/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
new file mode 100755
index 0000000..aa224fe
--- /dev/null
+++ b/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+cd /pulsar/pulsar-client-cpp
+
+find . -name CMakeCache.txt | xargs rm
+find . -name CMakeFiles | xargs rm -rf
+
+cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERSION \
+ -DPYTHON_LIBRARY=/opt/python/$PYTHON_SPEC/lib \
+ -DLINK_STATIC=ON
+
+make clean
+make _pulsar
+
+cd python
+python setup.py bdist_wheel
diff --git a/pulsar-client-cpp/docker/build-wheels.sh b/pulsar-client-cpp/docker/build-wheels.sh
new file mode 100755
index 0000000..a947a46
--- /dev/null
+++ b/pulsar-client-cpp/docker/build-wheels.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+set -e
+
+BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd $ROOT_DIR
+
+PYTHON_VERSIONS=(
+ '2.7 cp27-cp27mu'
+ '2.7 cp27-cp27m'
+ '3.3 cp33-cp33m'
+ '3.4 cp34-cp34m'
+ '3.5 cp35-cp35m'
+ '3.6 cp36-cp36m'
+)
+
+for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a PY <<< "$line"
+ PYTHON_VERSION=${PY[0]}
+ PYTHON_SPEC=${PY[1]}
+ echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC"
+
+ IMAGE_NAME=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+
+ echo "Using image: $IMAGE_NAME"
+ docker run -i -v $PWD:/pulsar $IMAGE_NAME /build.sh
+done
diff --git a/pulsar-client-cpp/docker/create-images.sh b/pulsar-client-cpp/docker/create-images.sh
new file mode 100755
index 0000000..b27f6e9
--- /dev/null
+++ b/pulsar-client-cpp/docker/create-images.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+# Create all the Docker images for variations of Python versions
+
+set -e
+
+PYTHON_VERSIONS=(
+ '2.7 cp27-cp27mu'
+ '2.7 cp27-cp27m'
+ '3.3 cp33-cp33m'
+ '3.4 cp34-cp34m'
+ '3.5 cp35-cp35m'
+ '3.6 cp36-cp36m'
+)
+
+for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a PY <<< "$line"
+ PYTHON_VERSION=${PY[0]}
+ PYTHON_SPEC=${PY[1]}
+ echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC"
+
+ IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+
+ docker build -t $IMAGE_NAME . \
+ --build-arg PYTHON_VERSION=$PYTHON_VERSION \
+ --build-arg PYTHON_SPEC=$PYTHON_SPEC
+
+ echo "==== Successfully built image $IMAGE_NAME"
+done
diff --git a/pulsar-client-cpp/docker/push-images.sh b/pulsar-client-cpp/docker/push-images.sh
new file mode 100755
index 0000000..5a542e0
--- /dev/null
+++ b/pulsar-client-cpp/docker/push-images.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+# Create all the Docker images for variations of Python versions
+
+set -e
+
+DOCKER_ORG=apachepulsar
+
+PYTHON_VERSIONS=(
+ '2.7 cp27-cp27mu'
+ '2.7 cp27-cp27m'
+ '3.3 cp33-cp33m'
+ '3.4 cp34-cp34m'
+ '3.5 cp35-cp35m'
+ '3.6 cp36-cp36m'
+)
+
+for line in "${PYTHON_VERSIONS[@]}"; do
+ read -r -a PY <<< "$line"
+ PYTHON_VERSION=${PY[0]}
+ PYTHON_SPEC=${PY[1]}
+
+ IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ FULL_NAME=$DOCKER_ORG/$IMAGE_NAME
+
+ echo "IMAGE_NAME: $IMAGE_NAME"
+ echo "FULL_NAME: $FULL_NAME"
+ docker tag $IMAGE_NAME $FULL_NAME
+ docker push $FULL_NAME
+
+ echo "==== Successfully pushed image $FULL_NAME"
+done
--
To stop receiving notification emails like this one, please contact
mmerli@apache.org.