You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2021/11/18 14:37:15 UTC
[yetus] branch main updated: YETUS-1138. Better support for multiple architectures (#238)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new ff9a461 YETUS-1138. Better support for multiple architectures (#238)
ff9a461 is described below
commit ff9a4612ed5ce540e0d2436ff419922ae9a1391d
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Thu Nov 18 06:37:07 2021 -0800
YETUS-1138. Better support for multiple architectures (#238)
---
asf-site-src/Dockerfile | 13 ++--
hooks/build | 59 +++++++++++++++++-
hooks/push | 5 +-
.../src/main/shell/test-patch-docker/Dockerfile | 72 ++++++++++------------
4 files changed, 99 insertions(+), 50 deletions(-)
diff --git a/asf-site-src/Dockerfile b/asf-site-src/Dockerfile
index 99f44bc..d720057 100644
--- a/asf-site-src/Dockerfile
+++ b/asf-site-src/Dockerfile
@@ -80,12 +80,11 @@ ENV GEM_HOME ${BUNDLE_PATH}
# on re-launches
# hadolint ignore=DL3028
RUN gem install bundler \
- middleman:'4.4.0' \
- middleman-livereload \
- middleman-syntax \
- nokogiri:1.10.10 \
- sassc:2.4.0 \
- therubyracer \
- tzinfo-data
+ middleman:'4.4.0' \
+ middleman-livereload \
+ middleman-syntax \
+ nokogiri:1.10.10 \
+ sassc:2.4.0 \
+ tzinfo-data
CMD ["/bin/bash"]
diff --git a/hooks/build b/hooks/build
index 0db14e3..b6a6b80 100755
--- a/hooks/build
+++ b/hooks/build
@@ -14,26 +14,79 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+if [[ -n "${DOCKER_MULTIARCH}" ]]; then
+
+ docker buildx create --name yetus-multiarch --driver docker-container --use || exit 1
+ docker buildx inspect --bootstrap || exit 1
+
+ traphandler() {
+ docker buildx rm yetus-multiarch || true
+ }
+
+ trap traphandler HUP INT QUIT TERM
+
+fi
+
GIT_URL=$(git config --get remote.origin.url)
# shellcheck disable=SC2034
DOCKER_BUILDKIT=1
export DOCKER_BUILDKIT
+# shellcheck disable=SC2034
+DOCKER_CLI_EXPERIMENTAL=enabled
+export DOCKER_CLI_EXPERIMENTAL
+
+if [[ -z "${SOURCE_COMMIT}" ]]; then
+ SOURCE_COMMIT="$(git rev-parse --verify HEAD)"
+ export SOURCE_COMMIT
+fi
+
+if [[ -z "${SOURCE_BRANCH}" ]]; then
+ SOURCE_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+ export SOURCE_BRANCH
+fi
+
+if [[ -z "${DOCKER_TAG}" ]]; then
+ DOCKER_TAG=${SOURCE_BRANCH}
+ export DOCKER_TAG
+fi
+
+if [[ -z "${DOCKER_REPO}" ]]; then
+ DOCKER_REPO=apache/yetus
+ export DOCKER_REPO
+fi
+
+KNOWN_PLATFORMS=$(docker buildx inspect --bootstrap)
+
+if [[ ${KNOWN_PLATFORMS} =~ linux/amd64 ]]; then
+ PLATFORMS+=(linux/amd64)
+fi
+
+if [[ ${KNOWN_PLATFORMS} =~ linux/arm64 ]]; then
+ PLATFORMS+=(linux/arm64)
+fi
+
+PLATSTRING=${PLATFORMS[*]}
+PLATSTRING=${PLATSTRING/ /,}
+echo "Building for ${PLATSTRING}"
+
# Build the -base image
-docker build \
+docker buildx build --platform "${PLATSTRING}" \
--label "git-commit-sha=${SOURCE_COMMIT}" \
--label "git-commit-branch=${SOURCE_BRANCH}" \
--label "git-commit-url=${GIT_URL}" \
--tag "${DOCKER_REPO}-base:${DOCKER_TAG}" \
- precommit/src/main/shell/test-patch-docker
+ --push \
+ precommit/src/main/shell/test-patch-docker || exit 1
# Build the full image using base above
-docker build \
+docker buildx build --platform "${PLATSTRING}" \
--label "git-commit-sha=${SOURCE_COMMIT}" \
--label "git-commit-branch=${SOURCE_BRANCH}" \
--label "git-commit-url=${GIT_URL}" \
--tag "${DOCKER_REPO}:${DOCKER_TAG}" \
--build-arg DOCKER_TAG="${DOCKER_TAG}" \
--build-arg DOCKER_REPO="${DOCKER_REPO}" \
+ --push \
.
diff --git a/hooks/push b/hooks/push
index 8452e68..f361d68 100755
--- a/hooks/push
+++ b/hooks/push
@@ -14,5 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-docker push "${DOCKER_REPO}-base:${DOCKER_TAG}"
-docker push "${DOCKER_REPO}:${DOCKER_TAG}"
\ No newline at end of file
+# push happens in build step now
+
+exit 0
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index 88823ac..cc11059 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -99,7 +99,7 @@ RUN add-apt-repository -y ppa:longsleep/golang-backports \
####
# Install Apache Creadur RAT jar
####
-FROM yetusbase AS apacherat
+FROM yetusbase AS yetusapacherat
ARG APACHE_RAT_VERSION=0.13
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN mkdir -p /opt/apache-rat \
@@ -114,7 +114,7 @@ RUN mkdir -p /opt/apache-rat \
#####
# Install SpotBugs
#####
-FROM yetusbase AS spotbugs
+FROM yetusbase AS yetusspotbugs
ARG SPOTBUGS_VERSION=4.4.1
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl -sSL https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/$SPOTBUGS_VERSION/spotbugs-$SPOTBUGS_VERSION.tgz -o spotbugs.tgz \
@@ -125,53 +125,48 @@ RUN curl -sSL https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/
&& tar -C /opt/spotbugs --strip-components 1 -xpf spotbugs.tgz \
&& rm spotbugs.tgz spotbugs.tgz.sha1 \
&& chmod a+rx /opt/spotbugs/bin/*
-ENV SPOTBUGS_HOME /opt/spotbugs
+## NOTE: SPOTBUGS_HOME is set below
####
# Install shellcheck (shell script lint)
####
-FROM yetusbase AS shellcheck
+FROM yetusbase AS yetusshellcheck
ARG SHELLCHECK_VERSION=0.7.2
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl -sSL \
- https://github.com/koalaman/shellcheck/releases/download/v$SHELLCHECK_VERSION/shellcheck-v$SHELLCHECK_VERSION.linux.x86_64.tar.xz \
+ https://github.com/koalaman/shellcheck/releases/download/v$SHELLCHECK_VERSION/shellcheck-v$SHELLCHECK_VERSION.linux."$(uname -m)".tar.xz \
| tar --strip-components 1 --wildcards -xJf - '*/shellcheck' \
&& chmod a+rx shellcheck \
- && mv shellcheck /bin/shellcheck \
- && shasum -a 512 /bin/shellcheck \
- | awk '$1!="0977b3d34f6e7e6d5c95ba46ca648ab55a6168e6304484d0daffefb0724c2309f9b0a2c8121d72ddcee977ba5614a5e7e83efc419b294600e0f73d748ec263a0" {exit(1)}'
+ && mv shellcheck /bin/shellcheck
####
# Install hadolint (dockerfile lint)
####
-FROM yetusbase AS hadolint
-ARG HADOLINT_VERSION=2.7.0
+FROM yetusbase AS yetushadolint
+ARG HADOLINT_VERSION=2.8.0
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN curl -sSL \
- https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VERSION/hadolint-Linux-x86_64 \
+RUN if [[ "$(uname -m)" == "x86_64" ]]; then curl -sSL \
+ https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VERSION/hadolint-Linux-"$(uname -m)" \
-o /bin/hadolint \
- && chmod a+rx /bin/hadolint \
- && shasum -a 512 /bin/hadolint \
- | awk '$1!="176eb3fd9f6a3cb9eda7f23b69a98069ea77cefc2078cf6234ebb3cb45897078934ee6925bec32b6e5c10915e8ff39bf24ebec36f37b226a2c8c4068a456ee30" {exit(1)}'
+ && chmod a+rx /bin/hadolint; \
+ else touch /bin/hadolint; fi
####
# Install buf (protobuf lint)
####
-FROM yetusbase AS buf
+FROM yetusbase AS yetusbuf
ARG BUF_VERSION=0.56.0
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl -sSL \
- https://github.com/bufbuild/buf/releases/download/v$BUF_VERSION/buf-Linux-x86_64.tar.gz \
+ https://github.com/bufbuild/buf/releases/download/v$BUF_VERSION/buf-Linux-"$(uname -m)".tar.gz \
-o buf.tar.gz \
- && shasum -a 512 buf.tar.gz \
- | awk '$1!="52b8ef7ba924b6679d59134df0765fe456749e83a9842ce289875489261b62b5261a21fc9849a22778a6afb3c5fcc6e86b856e06d1b62564d794c3e8f4da10ca" {exit(1)}' \
&& tar -xzf buf.tar.gz -C /usr/local --strip-components 1 \
&& rm buf.tar.gz
####
# Install bats (TAP-capable unit testing for shell scripts)
####
-FROM yetusbase AS bats
+FROM yetusbase AS yetusbats
ARG BATS_VERSION=1.4.1
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN git clone --branch v$BATS_VERSION \
@@ -180,11 +175,10 @@ RUN git clone --branch v$BATS_VERSION \
&& /tmp/bats-core/install.sh /opt/bats \
&& rm -rf /tmp/bats-core
-
####
# revive (golint on steroids)
####
-FROM yetusbase AS revive
+FROM yetusbase AS yetusrevive
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN go get -u github.com/mgechev/revive
@@ -195,7 +189,7 @@ RUN go get -u github.com/mgechev/revive
# requires go 1.15 as of 2021-03-02
#
####
-FROM yetusbase AS checkmake
+FROM yetusbase AS yetuscheckmake
ARG CHECKMAKE_VERSION=8915bd40945dd20b153cfcecb787ecdb5020ba38
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV BUILDER_NAME='Apache Yetus'
@@ -203,13 +197,13 @@ ENV BUILDER_EMAIL='dev@apache.yetus.org'
RUN git clone \
https://github.com/mrtazz/checkmake.git /tmp/checkmake \
&& git -C /tmp/checkmake checkout $CHECKMAKE_VERSION \
- && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 make -C /tmp/checkmake binaries \
+ && GOOS=linux CGO_ENABLED=0 make -C /tmp/checkmake binaries \
&& make -C /tmp/checkmake test
####
# golangci-lint (Multi-tool golang linter)
####
-FROM yetusbase as golangci
+FROM yetusbase as yetusgolangci
ARG GOLANGCILINT_VERSION=1.42.1
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN (GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v$GOLANGCILINT_VERSION)
@@ -246,10 +240,12 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8
RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-jre-headless openjdk-11-jdk-headless \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
- && update-java-alternatives -s java-1.8.0-openjdk-amd64 \
+ && update-java-alternatives -s "$(update-java-alternatives -l | grep 1.8.0 | awk '{print $1}')" \
&& rm -f /usr/lib/jvm/default-java \
- && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ && ln -s "$(update-java-alternatives -l | grep 1.8.0 | awk '{print $NF}')" /usr/lib/jvm/default-java
+# this var will get yetus_abs'd when run under precommit so should be relatively safe
+ENV JAVA_HOME /usr/lib/jvm/default-java
+ENV SPOTBUGS_HOME /opt/spotbugs
####
# Install ant
@@ -282,7 +278,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \
RUN curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
# hadolint ignore=DL3008,DL3059
RUN add-apt-repository -y \
- "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable" \
&& apt-get -q update && apt-get -q install --no-install-recommends -y docker-ce \
@@ -397,17 +393,17 @@ RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - \
# Now all the stuff that was built in parallel
#####
-COPY --from=apacherat /opt/apache-rat /opt/apache-rat
-COPY --from=spotbugs /opt/spotbugs /opt/spotbugs
-COPY --from=shellcheck /bin/shellcheck /bin/shellcheck
-COPY --from=hadolint /bin/hadolint /bin/hadolint
-COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
-COPY --from=bats /opt/bats /opt/bats
+COPY --from=yetusapacherat /opt/apache-rat /opt/apache-rat
+COPY --from=yetusspotbugs /opt/spotbugs /opt/spotbugs
+COPY --from=yetusshellcheck /bin/shellcheck /bin/shellcheck
+COPY --from=yetushadolint /bin/hadolint /bin/hadolint
+COPY --from=yetusbuf /usr/local/bin/buf /usr/local/bin/buf
+COPY --from=yetusbats /opt/bats /opt/bats
RUN ln -s /opt/bats/bin/bats /usr/local/bin/bats
-COPY --from=revive /root/go/bin/* /usr/local/bin
-COPY --from=checkmake /tmp/checkmake/checkmake /usr/local/bin
-COPY --from=golangci /root/go/bin/* /usr/local/bin
+COPY --from=yetusrevive /root/go/bin/* /usr/local/bin
+COPY --from=yetuscheckmake /tmp/checkmake/checkmake /usr/local/bin
+COPY --from=yetusgolangci /root/go/bin/* /usr/local/bin
####
# YETUS CUT HERE