You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/09/14 00:44:40 UTC
[pulsar] branch branch-2.7 updated: [Branch-2.7][Build] Use ubuntu
20.04 as docker image base (#12017)
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 259c698 [Branch-2.7][Build] Use ubuntu 20.04 as docker image base (#12017)
259c698 is described below
commit 259c698bdba20d68917d14591740f151084a1433
Author: ran <ga...@126.com>
AuthorDate: Tue Sep 14 08:43:37 2021 +0800
[Branch-2.7][Build] Use ubuntu 20.04 as docker image base (#12017)
### Motivation
Currently, the docker is from `openjdk:8-jdk-slim `, it has a new release a few days ago and the Linux version was changed to `Impish`, it didn't support install `python3.7` by apt-get tool, the minimum Python version is 3.9, so we need to change the python version or use a different Linux release, I cherry-pick some commits from branch master to build docker image from the `ubuntu:20.04`.
Mainly related PRs:
1. https://github.com/apache/pulsar/pull/11026
2. https://github.com/apache/pulsar/pull/11623
3. https://github.com/apache/pulsar/pull/11862
### Modifications
Build docker image from the `ubuntu:20.04`.
upgrade pulsar-go-client from `0.2.0` to `0.6.0` and rever #9124 from branch-2.7
---
.github/workflows/ci-docker-build.yaml | 90 ++++++++++++++++++++++
docker/pulsar-standalone/Dockerfile | 13 +++-
docker/pulsar/Dockerfile | 27 +++----
docker/pulsar/pom.xml | 4 +-
...ulsar-client-37.sh => install-pulsar-client.sh} | 5 +-
pulsar-function-go/go.mod | 2 +-
pulsar-function-go/pf/instance.go | 11 ---
7 files changed, 119 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/ci-docker-build.yaml b/.github/workflows/ci-docker-build.yaml
new file mode 100644
index 0000000..e69b732
--- /dev/null
+++ b/.github/workflows/ci-docker-build.yaml
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: CI - Docker Build
+on:
+ pull_request:
+ branches:
+ - master
+ - branch-*
+ push:
+ branches:
+ - branch-*
+
+env:
+ MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+
+ sql:
+ name:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 25
+ ref: ${{ github.event.pull_request.head.sha }}
+
+ - name: Cache local Maven repository
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+
+ - name: Check if this pull request only changes documentation
+ id: docs
+ uses: apache/pulsar-test-infra/diff-only@master
+ with:
+ args: site2 deployment .asf.yaml .ci ct.yaml
+
+ - name: Set up JDK 1.8
+ uses: actions/setup-java@v1
+ if: steps.docs.outputs.changed_only == 'no'
+ with:
+ java-version: 1.8
+
+ - name: clean disk
+ if: steps.docs.outputs.changed_only == 'no'
+ run: |
+ sudo swapoff -a
+ sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
+ sudo apt clean
+ docker rmi $(docker images -q) -f
+ df -h
+
+ - name: run install by skip tests
+ if: steps.docs.outputs.changed_only == 'no'
+ run: mvn -q -B -ntp clean install -DskipTests
+
+ - name: build pulsar image
+ if: steps.docs.outputs.changed_only == 'no'
+ run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+ - name: build pulsar-all image
+ if: steps.docs.outputs.changed_only == 'no'
+ run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+ - name: build artifacts and docker pulsar latest test image
+ if: steps.docs.outputs.changed_only == 'no'
+ run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests
diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile
index 0035de5..600f747 100644
--- a/docker/pulsar-standalone/Dockerfile
+++ b/docker/pulsar-standalone/Dockerfile
@@ -24,16 +24,21 @@ FROM apachepulsar/pulsar-all:latest as pulsar
FROM apachepulsar/pulsar-dashboard:latest as dashboard
# Restart from
-FROM openjdk:8-jdk
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+ && apt-get -y install wget gnupg && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN bash -c "echo deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main >> /etc/apt/sources.list.d/pgdg.list"
# Note that the libpq-dev package is needed here in order to install
# the required python psycopg2 package (for postgresql) later
RUN apt-get update \
- && apt-get -y install python3.7 python3.7-dev python3-pip postgresql sudo nginx supervisor libpq-dev
+ && apt-get -y install openjdk-11-jdk-headless python3 python3-dev python3-pip postgresql-11 sudo nginx supervisor libpq-dev
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
RUN update-ca-certificates
-RUN ln -s /usr/bin/pip3 /usr/bin/pip
# Postgres configuration
COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/11/main/postgresql.conf
@@ -73,7 +78,7 @@ RUN pip install -r /pulsar/django/requirements.txt
COPY --from=dashboard /pulsar/init-postgres.sh /pulsar/django/init-postgres.sh
RUN mkdir /data
RUN /pulsar/django/init-postgres.sh
-RUN sudo -u postgres /etc/init.d/postgresql stop
+RUN sudo -u postgres /etc/init.d/postgresql stop
# Add postgresql to supervisord. Redirect logs to stdout
RUN echo "\n[program:postgresql]\n\
command = /etc/init.d/postgresql start\n\
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 37a022d..c5c20bd 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -32,36 +32,37 @@ COPY scripts/generate-zookeeper-config.sh /pulsar/bin
COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin
COPY scripts/watch-znode.py /pulsar/bin
COPY scripts/set_python_version.sh /pulsar/bin
-COPY scripts/install-pulsar-client-37.sh /pulsar/bin
+COPY scripts/install-pulsar-client.sh /pulsar/bin
-### Create 2nd stage from OpenJDK image
-### and add Python dependencies (for Pulsar functions)
+### Create 2nd stage from Ubuntu image
+### and add OpenJDK and Python dependencies (for Pulsar functions)
-FROM openjdk:8-jdk-slim
+FROM ubuntu:20.04
-# Install software-properties-common
-RUN apt update \
- && apt install software-properties-common \
- && add-apt-repository ppa:deadsnakes/ppa
+ARG DEBIAN_FRONTEND=noninteractive
# Install some utilities
RUN apt-get update \
- && apt-get install -y netcat dnsutils less procps iputils-ping \
- python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \
+ && apt-get -y dist-upgrade \
+ && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \
+ python3 python3-dev python3-setuptools python3-yaml python3-kazoo \
libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
curl \
+ && apt-get -y --purge autoremove \
+ && apt-get autoclean \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.7 get-pip.py
+RUN python3 get-pip.py
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
ADD target/python-client/ /pulsar/pulsar-client
ADD target/cpp-client/ /pulsar/cpp-client
-RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security
RUN apt-get update \
&& apt install -y /pulsar/cpp-client/*.deb \
&& apt-get clean \
@@ -75,4 +76,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
COPY --from=pulsar /pulsar /pulsar
WORKDIR /pulsar
-RUN /pulsar/bin/install-pulsar-client-37.sh
+RUN /pulsar/bin/install-pulsar-client.sh
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 1cc5861..58f1390 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -63,8 +63,8 @@
<workingDirectory>${project.basedir}/target</workingDirectory>
<executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
<arguments>
- <!-- build python 3.7 -->
- <argument>3.7 cp37-cp37m</argument>
+ <!-- build python 3.8 -->
+ <argument>3.8 cp38-cp38</argument>
</arguments>
</configuration>
</execution>
diff --git a/docker/pulsar/scripts/install-pulsar-client-37.sh b/docker/pulsar/scripts/install-pulsar-client.sh
similarity index 79%
rename from docker/pulsar/scripts/install-pulsar-client-37.sh
rename to docker/pulsar/scripts/install-pulsar-client.sh
index 9d3714c..56702b1 100755
--- a/docker/pulsar/scripts/install-pulsar-client-37.sh
+++ b/docker/pulsar/scripts/install-pulsar-client.sh
@@ -20,5 +20,6 @@
set -x
-WHEEL_FILE=$(ls /pulsar/pulsar-client | grep cp37)
-pip3.7 install /pulsar/pulsar-client/${WHEEL_FILE}
+PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/')
+WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}")
+pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all]
diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod
index 5608345..6170ae9 100644
--- a/pulsar-function-go/go.mod
+++ b/pulsar-function-go/go.mod
@@ -3,7 +3,7 @@ module github.com/apache/pulsar/pulsar-function-go
go 1.13
require (
- github.com/apache/pulsar-client-go v0.2.0
+ github.com/apache/pulsar-client-go v0.6.0
github.com/golang/protobuf v1.4.2
github.com/prometheus/client_golang v1.7.1
github.com/prometheus/client_model v0.2.0
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index afbf299..d3a45e0 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -224,17 +224,6 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) {
gi.context.instanceConf.funcDetails.Namespace,
gi.context.instanceConf.funcDetails.Name), gi.context.instanceConf.instanceID)
- batchBuilderType := pulsar.DefaultBatchBuilder
-
- if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil {
- batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder
- if batchBuilder != "" {
- if batchBuilder == "KEY_BASED" {
- batchBuilderType = pulsar.KeyBasedBatchBuilder
- }
- }
- }
-
producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{
Topic: topicName,
Properties: properties,