You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/04/05 02:47:33 UTC
[pinot] branch master updated: Adding Github Actions to build Pinot/Presto/Superset docker images (#8458)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 164b7a4cd0 Adding Github Actions to build Pinot/Presto/Superset docker images (#8458)
164b7a4cd0 is described below
commit 164b7a4cd00967c662508ae477fdb40a0941ea3d
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Mon Apr 4 19:47:26 2022 -0700
Adding Github Actions to build Pinot/Presto/Superset docker images (#8458)
---
.github/workflows/build-pinot-docker-image.yml | 69 +++++++++++++++++++++
.github/workflows/build-presto-docker-image.yml | 70 ++++++++++++++++++++++
.github/workflows/build-superset-docker-image.yml | 65 ++++++++++++++++++++
.../docker/.pinot_docker_image_build_and_push.sh | 61 +++++++++++++++++++
.../docker/.presto_docker_image_build_and_push.sh | 70 ++++++++++++++++++++++
.../.superset_docker_image_build_and_push.sh | 59 ++++++++++++++++++
6 files changed, 394 insertions(+)
diff --git a/.github/workflows/build-pinot-docker-image.yml b/.github/workflows/build-pinot-docker-image.yml
new file mode 100644
index 0000000000..13a2163f7b
--- /dev/null
+++ b/.github/workflows/build-pinot-docker-image.yml
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Docker Image Build and Publish
+
+on:
+ schedule:
+ - cron: '0 1 * * *' # run at 1 AM UTC
+
+ workflow_dispatch:
+ inputs:
+ gitUrl:
+ description: "The Pinot git repo to check out to build, use https."
+ default: "https://github.com/apache/pinot.git"
+ required: true
+ commit:
+ description: "The branch/commit to check out to build Pinot image."
+ default: "master"
+ required: true
+ dockerImageName:
+ description: "The docker image name, default to 'apachepinot/pinot'."
+ default: "apachepinot/pinot"
+ required: true
+ tags:
+ description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+ default: ""
+
+jobs:
+ build-pinot-docker-image:
+ name: Build Pinot Docker Image
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ platform: [ "linux/amd64", "linux/arm64" ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v1
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v1
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ DOCKER_FILE_BASE_DIR: "docker/images/pinot"
+ DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+ BUILD_PLATFORM: ${{ matrix.platform }}
+ PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PINOT_BRANCH: ${{ github.event.inputs.commit }}
+ TAGS: ${{ github.event.inputs.tags }}
+ run: .github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh
diff --git a/.github/workflows/build-presto-docker-image.yml b/.github/workflows/build-presto-docker-image.yml
new file mode 100644
index 0000000000..40aafd4b3e
--- /dev/null
+++ b/.github/workflows/build-presto-docker-image.yml
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Presto Pinot Docker Image Build and Publish
+
+on:
+ schedule:
+ - cron: '0 0 * * *' # run at 0 AM UTC
+
+ workflow_dispatch:
+ inputs:
+ gitUrl:
+ description: "The git url to check out to build Presto, use https not ssh."
+ default: "https://github.com/prestodb/presto.git"
+ required: true
+ commit:
+ description: "The branch/commit to check out to build Presto image.."
+ default: "master"
+ required: true
+ platform:
+ description: "The platform of the image to build, e.g. linux/amd64"
+ default: "linux/amd64"
+ required: true
+ dockerImageName:
+ description: "The docker image name, default is 'apachepinot/pinot-presto'"
+ default: "apachepinot/pinot-presto"
+ required: true
+ tags:
+ description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+ default: ""
+
+jobs:
+ build-presto-pinot-docker-image:
+ name: Build Presto Pinot Docker Image
+ runs-on: ubuntu-latest
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v1
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v1
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ DOCKER_FILE_BASE_DIR: "docker/images/pinot-presto"
+ DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+ BUILD_PLATFORM: ${{ github.event.inputs.platform }}
+ PRESTO_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PRESTO_BRANCH: ${{ github.event.inputs.commit }}
+ TAGS: ${{ github.event.inputs.tags }}
+ run: .github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh
diff --git a/.github/workflows/build-superset-docker-image.yml b/.github/workflows/build-superset-docker-image.yml
new file mode 100644
index 0000000000..e073a5a9b9
--- /dev/null
+++ b/.github/workflows/build-superset-docker-image.yml
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Superset Pinot Docker Image Build and Publish
+
+on:
+ schedule:
+ - cron: '0 0 * * *' # run at 0 AM UTC
+
+ workflow_dispatch:
+ inputs:
+ supersetImageTag:
+ description: "Base superset docker image tag from https://hub.docker.com/r/apache/superset"
+ default: "latest"
+ required: true
+ platform:
+ description: "The platform of the image to build, e.g. linux/amd64"
+ default: "linux/amd64"
+ required: true
+ dockerImageName:
+ description: "The docker image name, default is 'apachepinot/pinot-superset'"
+ default: "apachepinot/pinot-superset"
+ required: true
+ tags:
+ description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+ default: ""
+
+jobs:
+ build-superset-pinot-docker-image:
+ name: Build Superset Pinot Docker Image
+ runs-on: ubuntu-latest
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v1
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v1
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ DOCKER_FILE_BASE_DIR: "docker/images/pinot-superset"
+ DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+ SUPERSET_IMAGE_TAG: ${{ github.event.inputs.supersetImageTag }}
+ BUILD_PLATFORM: ${{ github.event.inputs.platform }}
+ TAGS: ${{ github.event.inputs.tags }}
+ run: .github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh
diff --git a/.github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh b/.github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh
new file mode 100755
index 0000000000..f7f46e8a60
--- /dev/null
+++ b/.github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh
@@ -0,0 +1,61 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+if [ -z "${PINOT_GIT_URL}" ]; then
+ PINOT_GIT_URL="https://github.com/apache/pinot.git"
+fi
+if [ -z "${PINOT_BRANCH}" ]; then
+ PINOT_BRANCH="master"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+ BUILD_PLATFORM="linux/arm64,linux/amd64"
+fi
+
+COMMIT_ID=`git rev-parse --short HEAD`
+DATE=`date +%Y%m%d`
+VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
+tags=()
+if [ -z "${TAGS}" ]; then
+ tags=("${VERSION}-${COMMIT_ID}-${DATE}")
+ tags+=("latest")
+else
+ declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+ echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+ --no-cache \
+ --platform=${BUILD_PLATFORM} \
+ --file Dockerfile \
+ --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} --build-arg PINOT_BRANCH=${PINOT_BRANCH} \
+ ${DOCKER_BUILD_TAGS} \
+ --push \
+ .
diff --git a/.github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh b/.github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh
new file mode 100755
index 0000000000..9db8baf612
--- /dev/null
+++ b/.github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh
@@ -0,0 +1,70 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot-presto"
+fi
+if [ -z "${PRESTO_GIT_URL}" ]; then
+ PRESTO_GIT_URL="https://github.com/prestodb/presto.git"
+fi
+if [ -z "${PRESTO_BRANCH}" ]; then
+ PINOT_BRANCH="master"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+ BUILD_PLATFORM="linux/amd64"
+fi
+
+# Get presto commit id
+ROOT_DIR=`pwd`
+rm -rf /tmp/presto
+git clone -b ${PRESTO_BRANCH} --single-branch ${PRESTO_GIT_URL} /tmp/presto
+cd /tmp/presto
+COMMIT_ID=`git rev-parse --short HEAD`
+VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
+rm -rf /tmp/presto
+DATE=`date +%Y%m%d`
+cd ${ROOT_DIR}
+
+tags=()
+if [ -z "${TAGS}" ]; then
+ tags=("${VERSION}-${COMMIT_ID}-${DATE}")
+ tags+=("latest")
+else
+ declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+ echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+ --no-cache \
+ --platform=${BUILD_PLATFORM} \
+ --file Dockerfile \
+ --build-arg PRESTO_GIT_URL=${PRESTO_GIT_URL} --build-arg PRESTO_BRANCH=${PRESTO_BRANCH} \
+ ${DOCKER_BUILD_TAGS} \
+ --push \
+ .
diff --git a/.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh b/.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh
new file mode 100755
index 0000000000..700b8e01a1
--- /dev/null
+++ b/.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh
@@ -0,0 +1,59 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot-superset"
+fi
+if [ -z "${SUPERSET_IMAGE_TAG}" ]; then
+ SUPERSET_IMAGE_TAG="latest"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+ BUILD_PLATFORM="linux/amd64"
+fi
+
+DATE=`date +%Y%m%d`
+docker pull apache/superset:${SUPERSET_IMAGE_TAG}
+COMMIT_ID=`docker images apache/superset:${SUPERSET_IMAGE_TAG} --format "{{.ID}}"`
+
+tags=()
+if [ -z "${TAGS}" ]; then
+ tags=("${COMMIT_ID}-${DATE}")
+ tags+=("latest")
+else
+ declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+ echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+ --no-cache \
+ --platform=${BUILD_PLATFORM} \
+ --file Dockerfile \
+ --build-arg SUPERSET_IMAGE_TAG=${SUPERSET_IMAGE_TAG} \
+ ${DOCKER_BUILD_TAGS} \
+ --push \
+ .
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org