You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2021/11/22 05:33:53 UTC
[yetus] branch main updated: YETUS-1142. precommit should pull from ghcr for better multi platform… (#240)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new a1975cf YETUS-1142. precommit should pull from ghcr for better multi platform… (#240)
a1975cf is described below
commit a1975cf560190c1acaccb53bf6383eb0ae0cb505
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Sun Nov 21 21:33:48 2021 -0800
YETUS-1142. precommit should pull from ghcr for better multi platform… (#240)
---
.buildkite/pipeline.yml | 2 +-
.circleci/config.yml | 2 +-
.semaphore/semaphore-build.sh | 2 +-
Jenkinsfile | 2 +-
asf-site-src/source/yetus-docker-image.html.md | 15 ++++---
precommit/src/main/shell/core.d/docker.sh | 7 +--
start-build-env.sh | 61 +++++++++++++++++++++++---
7 files changed, 71 insertions(+), 20 deletions(-)
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8b9df1d..fd6f6d5 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -23,7 +23,7 @@ steps:
--bugcomments=briefreport,htmlout,buildkiteannotate
--console-report-file=/tmp/yetus-out/console.txt
--docker
- --docker-cache-from=apache/yetus:main
+ --docker-cache-from=ghcr.io/apache/yetus-base:main
--html-report-file=/tmp/yetus-out/report.html
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
--patch-dir=/tmp/yetus-out
diff --git a/.circleci/config.yml b/.circleci/config.yml
index a291f46..7666fb0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -19,7 +19,7 @@ version: 2
jobs:
build:
docker:
- - image: apache/yetus:main
+ - image: ghcr.io/apache/yetus:main
working_directory: ~/repo
diff --git a/.semaphore/semaphore-build.sh b/.semaphore/semaphore-build.sh
index 879c661..ef151ab 100755
--- a/.semaphore/semaphore-build.sh
+++ b/.semaphore/semaphore-build.sh
@@ -25,4 +25,4 @@ PRECOMMITDIR=precommit/src/main/shell
--tests-filter=checkstyle,test4tests \
--docker \
--dockerfile="${PRECOMMITDIR}/test-patch-docker/Dockerfile" \
- --docker-cache-from=apache/yetus-base:main,ubuntu:focal
+ --docker-cache-from=ghcr.io/apache/yetus-base:main,ubuntu:focal
diff --git a/Jenkinsfile b/Jenkinsfile
index 9acb826..250191b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -135,7 +135,7 @@ pipeline {
docker pull ubuntu:focal
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${YETUS_DOCKERFILE}")
- YETUS_ARGS+=("--docker-cache-from=apache/yetus-base:main")
+ YETUS_ARGS+=("--docker-cache-from=ghcr.io/apache/yetus-base:main")
else
# need to figure this out programmatically; hard-coded for now
export JAVA_HOME=/home/jenkins/tools/java/latest1.8
diff --git a/asf-site-src/source/yetus-docker-image.html.md b/asf-site-src/source/yetus-docker-image.html.md
index 1d99c4b..31f30bb 100644
--- a/asf-site-src/source/yetus-docker-image.html.md
+++ b/asf-site-src/source/yetus-docker-image.html.md
@@ -26,13 +26,14 @@
<!-- /MarkdownTOC -->
-While not official releases, the Apache Yetus project makes available two types of images on hub.docker.com:
+While not official releases, the Apache Yetus project makes available two types of images available via
+the GitHub Container Registry:
-* apache/yetus-base
+* ghcr.io/apache/yetus-base
This image is the same as the 'built-in' Dockerfile when `--docker` is used without `--dockerfile` on the precommit command line. It includes all of the pre-requisites as needed by the various Apache Yetus components. It is located in `precommit/src/main/shell/test-patch-docker/`.
-* apache/yetus
+* ghcr.io/apache/yetus
This image is the same as apache/yetus-base but includes a pre-built version of Apache Yetus as part of the base OS image. In other words, qbt, releasedocmaker, shelldocs, test-patch, etc., are in /usr/bin and available in the default path. It is generated from the Dockerfile located in the root of the source and is built with the options provided in the hooks directory.
@@ -49,7 +50,7 @@ docker run \
--rm \
-v /tmp/out:/output \
-v /src/precommit/src/main/shell:/input:ro \
- apache/yetus:0.9.0 \
+ ghcr.io/apache/yetus:0.14.0 \
shelldocs --output /output/test-patch.md --input /input/test-patch.sh
```
@@ -62,7 +63,7 @@ docker run \
--rm \
-v /tmp/patchdir:/patchdir \
-v /src:/basedir \
- apache/yetus:0.9.0 \
+ ghcr.io/apache/yetus:0.14.0 \
qbt \
--plugins=all \
--basedir=/basedir \
@@ -73,7 +74,7 @@ docker run \
If your project needs additional dependencies, it is trivial to build off of the Apache Yetus image:
```Dockerfile
-FROM apache/yetus:0.9.0
+FROM ghcr.io/apache/yetus:0.14.0
RUN apt-get -q update && apt-get -q install -y \
clang \
libssl-dev \
@@ -85,7 +86,7 @@ RUN apt-get -q update && apt-get -q install -y \
docker build -t project/build:0.9.0 -f .
```
-This example builds a docker image based off of Apache Yetus 0.9.0 but with the additions of clang, some development libraries, and valgrind. Now project/build:0.9.0 can be used instead of apache/yetus:0.9.0 since it has all of Apache Yetus and the additions needed by our project.
+This example builds a docker image based off of Apache Yetus 0.9.0 but with the additions of clang, some development libraries, and valgrind. Now project/build:0.9.0 can be used instead of ghcr.io/apache/yetus:0.14.0 since it has all of Apache Yetus and the additions needed by our project.
# A Note About Precommit
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index cec18b9..a6edb2c 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -32,6 +32,7 @@ DOCKER_IN_DOCKER=false
DOCKER_SOCKET="/var/run/docker.sock"
DOCKER_SOCKET_GID=-1
DOCKER_WORK_DIR="/precommit"
+DOCKER_ASF_BUILT_REPO="ghcr.io/apache/yetus"
declare -a DOCKER_EXTRAARGS
declare -a DOCKER_EXTRABUILDARGS
@@ -261,11 +262,11 @@ function docker_fileverify
dockplat=('--platform' "${DOCKER_PLATFORM}")
fi
- echo "No --dockerfile or --docker-tag provided. Attempting to pull apache/yetus:${VERSION}."
+ echo "No --dockerfile or --docker-tag provided. Attempting to pull ${DOCKER_ASF_BUILT_REPO}:${VERSION}."
- if dockercmd pull "${dockplat[@]}" "apache/yetus:${VERSION}"; then
+ if dockercmd pull "${dockplat[@]}" "${DOCKER_ASF_BUILT_REPO}:${VERSION}"; then
echo "Pull succeeded; will build with pulled image."
- DOCKER_TAG="apache/yetus:${VERSION}"
+ DOCKER_TAG="${DOCKER_ASF_BUILT_REPO}:${VERSION}"
else
echo "Pull failed; will build with built-in Dockerfile."
DOCKERFILE=${DOCKERFILE_DEFAULT}
diff --git a/start-build-env.sh b/start-build-env.sh
index 3e5bea4..daaccd9 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -15,18 +15,58 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# SHELLDOC-IGNORE
+
+make_cache_list() {
+ gotit="false"
+ cache_array=()
+ for imagelocation in "${YETUS_DOCKER_REPO}" "${ASF_DOCKER_REPO}"; do
+ if [[ "${imagelocation}" == "apache/yetus" ]]; then
+ # skip Apache docker hub since we will pull from
+ # github later
+ continue
+ fi
+ for branch in "${BRANCH}" "main"; do
+ for type in "-base" ""; do
+ image="${imagelocation}${type}:${branch}"
+ if docker pull "${image}"; then
+ cache_array+=("${image}")
+ gotit="true"
+ break
+ fi
+ done
+ if [[ "${gotit}" == "true" ]]; then
+ gotit="false"
+ break
+ fi
+ done
+ done
+ printf -v thelist "%s," "${cache_array[@]}"
+ CACHE_LIST=${thelist%,}
+}
+
set -e # exit on error
ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")" >/dev/null && pwd -P)
+ASF_DOCKER_REPO="ghcr.io/apache/yetus"
YETUS_DOCKER_REPO=${YETUS_DOCKER_REPO:-apache/yetus}
+CACHE_LIST=""
# shellcheck disable=SC2034
DOCKER_BUILDKIT=1
export DOCKER_BUILDKIT
+# shellcheck disable=SC2034
+DOCKER_CLI_EXPERIMENTAL=1
+export DOCKER_CLI_EXPERIMENTAL
+
# moving to the path of the Dockerfile reduces the context
cd "${ROOTDIR}/precommit/src/main/shell/test-patch-docker"
+printf "Using:\n\n\n"
+docker version
+printf "\n\n\n"
+
BRANCH=$(git branch | grep '\*' | cut -d ' ' -f2 )
if [[ "${BRANCH}" =~ HEAD ]]; then
BRANCH=$(git branch | grep '\*' | awk '{print $NF}' | sed -e s,rel/,,g -e s,\),,g )
@@ -37,19 +77,28 @@ if [[ "${GITHUB_ACTIONS}" == true ]]; then
echo "::group::start-build-env - warm docker cache"
fi
-echo "Attempting a few pulls of ${YETUS_DOCKER_REPO} and ${YETUS_DOCKER_REPO}-base to save time"
+echo "Attempting a few pulls to save time"
echo "Errors here will be ignored!"
-docker pull "${YETUS_DOCKER_REPO}-base:${BRANCH}" || docker pull "${YETUS_DOCKER_REPO}-base:main" || true
-docker pull "${YETUS_DOCKER_REPO}:${BRANCH}" || docker pull "${YETUS_DOCKER_REPO}:main" || true
+
+make_cache_list
if [[ "${GITHUB_ACTIONS}" == true ]]; then
echo "::endgroup::"
echo "::group::start-build-env - rebuild base"
fi
-docker build \
- --cache-from="${YETUS_DOCKER_REPO}-build:${BRANCH},${YETUS_DOCKER_REPO}-base:${BRANCH},${YETUS_DOCKER_REPO}-base:main,${YETUS_DOCKER_REPO}:${BRANCH},${YETUS_DOCKER_REPO}:main" \
- -t "${YETUS_DOCKER_REPO}-build:${BRANCH}" .
+if [[ -n "${CACHE_LIST}" ]]; then
+ set -x
+ docker build \
+ --cache-from="${CACHE_LIST}" \
+ -t "${YETUS_DOCKER_REPO}-build:${BRANCH}" .
+ set +x
+else
+ set -x
+ docker build \
+ -t "${YETUS_DOCKER_REPO}-build:${BRANCH}" .
+ set +x
+fi
USER_NAME=${SUDO_USER:=$USER}
USER_ID=$(id -u "${USER_NAME}")