You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2020/02/29 02:14:56 UTC
[yetus] branch master updated: YETUS-947. Support github actions as
a robot (#95)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/master by this push:
new 0dc9e47 YETUS-947. Support github actions as a robot (#95)
0dc9e47 is described below
commit 0dc9e47fa4554ecbe6f8b5e040b64c724529b583
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Fri Feb 28 18:14:50 2020 -0800
YETUS-947. Support github actions as a robot (#95)
---
.github/workflows/yetus.yml | 51 +++++++++++++
.../in-progress/precommit-basic.html.md | 2 +-
.../in-progress/precommit-robots.html.md | 14 +++-
precommit/src/main/shell/core.d/00-yetuslib.sh | 2 +-
precommit/src/main/shell/core.d/docker.sh | 55 ++++++++------
precommit/src/main/shell/robots.d/githubactions.sh | 85 ++++++++++++++++++++++
precommit/src/main/shell/test-patch.d/github.sh | 31 +++++---
precommit/src/main/shell/test-patch.sh | 1 +
8 files changed, 203 insertions(+), 38 deletions(-)
diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml
new file mode 100644
index 0000000..f6e76d2
--- /dev/null
+++ b/.github/workflows/yetus.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+name: Apache Yetus
+
+on: [push, pull_request] # yamllint disable-line rule:truthy
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ path: src
+ - name: test-patch
+ run: >-
+ "${GITHUB_WORKSPACE}/src/precommit/src/main/shell/test-patch.sh"
+ --basedir="${GITHUB_WORKSPACE}/src"
+ --patch-dir="${GITHUB_WORKSPACE}/out"
+ --docker
+ --project=yetus
+ --brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt"
+ --console-report-file="${GITHUB_WORKSPACE}/out/console.txt"
+ --html-report-file="${GITHUB_WORKSPACE}/out/report.html"
+ --junit-report-xml="${GITHUB_WORKSPACE}/out/junit-report.xml"
+ --linecomments=''
+ --proclimit=2000
+ --archive-list="checkstyle-errors.xml,spotbugsXml.xml"
+ --plugins=all
+ --tests-filter=checkstyle,javadoc,rubocop,test4tests
+ --docker-cache-from=apache/yetus-base:master
+ - name: Artifact output
+ uses: actions/upload-artifact@v1
+ with:
+ name: apacheyetuspatchdir
+ path: out
diff --git a/asf-site-src/source/documentation/in-progress/precommit-basic.html.md b/asf-site-src/source/documentation/in-progress/precommit-basic.html.md
index 004782a..e38bc6c 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-basic.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-basic.html.md
@@ -97,7 +97,7 @@ Build Tools:
Automation and Isolation:
* [Circle CI](https://www.circleci.com)
-* [Docker](https://www.docker.com) version 1.7.0+
+* [Docker](https://www.docker.com) Minimum: 17.0+ (moby Client API 1.27.0+) - Recommended: 18.0+ (moby Client API 1.38.0+)
* [Gitlab CI](https://www.gitlab.com)
* [Jenkins](https://www.jenkins-ci.org)
* [Travis CI](https://www.travis-ci.com)
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.html.md b/asf-site-src/source/documentation/in-progress/precommit-robots.html.md
index b0fa588..dd50b75 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-robots.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.html.md
@@ -55,8 +55,6 @@ TRIGGER: ${TF_BUILD}=True
Azure Pipelines support has only been tested on the Ubuntu VM with GitHub as the source repository. It automatically configures `--patch-dir` to be `${BUILD_ARTIFACTSTAGINGDIRECTORY}/yetus`. While the URL to the console is provided in the report, links are not provided due to the URLs to artifacts not being available at runtime.
-As of this writing, Azure Pipelines has moved to a custom moby build for the 'docker' executable. As a result, `--docker` is not supported.
-
# Circle CI
TRIGGER: ${CIRCLECI}=true
@@ -135,6 +133,18 @@ See also:
* Apache Yetus' source tree [.cirrus.yml](https://github.com/apache/yetus/blob/master/.cirrus.yml) for some tips and tricks.
+# GitHub Actions
+
+ NOTE: GitHub Actions support is not stable and should be viewed as experimental, at best.
+
+TRIGGER: ${GITHUB_ACTIONS}=True
+
+GitHub Actions support has only been tested on the ubuntu-latest image. It automatically configures `--patch-dir` to be `${GITHUB_WORKSAPCE}/yetus` if not previously set.
+
+See also:
+
+* Apache Yetus' source tree [yetus.yaml](https://github.com/apache/yetus/blob/master/.github/workflows/yetus.yml) for some tips and tricks.
+
# Gitlab CI
TRIGGER: ${GITLAB_CI}=true
diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh b/precommit/src/main/shell/core.d/00-yetuslib.sh
index 780dc2a..0cb2e15 100755
--- a/precommit/src/main/shell/core.d/00-yetuslib.sh
+++ b/precommit/src/main/shell/core.d/00-yetuslib.sh
@@ -180,7 +180,7 @@ function yetus_generic_columnprinter
if [[ -n "${COLUMNS}" ]]; then
numcols=${COLUMNS}
else
- numcols=$(tput cols) 2>/dev/null
+ numcols=$(tput cols 2>/dev/null)
fi
if [[ -z "${numcols}"
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index e47cea4..4d210e9 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -146,7 +146,7 @@ function docker_parse_args
;;
--docker-work-dir=*)
delete_parameter "${i}"
- DOCKER_TAG=${i#*=}
+ DOCKER_WORK_DIR=${i#*=}
;;
esac
done
@@ -196,8 +196,9 @@ function docker_initialize
dockvers=$(docker_version Client)
IFS='.' read -r -a DOCKER_VERSION <<< "${dockvers}"
- if [[ "${DOCKER_VERSION[0]}" -lt 17 ]]; then
- add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 17.0)."
+ if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \
+ [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 27 ]]; then
+ add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < API v 1.27.0)."
bugsystem_finalreport 1
cleanup_and_exit 1
fi
@@ -247,7 +248,7 @@ function docker_fileverify
dockplat=('--platform' "${DOCKER_PLATFORM}")
fi
- echo "No --dockerfile or --dockertag provided. Attempting to pull apache/yetus:${VERSION}."
+ echo "No --dockerfile or --docker-tag provided. Attempting to pull apache/yetus:${VERSION}."
if dockercmd pull "${dockplat[@]}" "apache/yetus:${VERSION}"; then
echo "Pull succeeded; will build with pulled image."
@@ -559,7 +560,7 @@ function docker_version
declare ret
# new version command
- val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null)
+ val=$(dockercmd version --format "{{.${vertype}.APIVersion}}" 2>/dev/null)
ret=$?
if [[ ${ret} != 0 ]];then
@@ -676,20 +677,23 @@ function docker_run_image
lines=$("${AWK}" '/YETUS CUT HERE/ {print FNR; exit}' "${DOCKERFILE}")
buildfile="${PATCH_DIR}/Dockerfile"
- if [[ "${DOCKER_VERSION[0]}" -lt 18 ]] && [[ ${lines} -gt 0 ]]; then
-
- # versions less than 18 don't support having the
- # Dockerfile outside of the build context. Let's fall back to
- # pre-YETUS-723 behavior and put the re-written Dockerfile
- # outside of the source tree rather than go through a lot of
- # machinations. This means COPY, ADD, etc do not work, but
- # whatever
-
- popd >/dev/null || return 1
- buildfile="${PATCH_DIR}/test-patch-docker/Dockerfile"
- dockerdir="${PATCH_DIR}/test-patch-docker"
- mkdir -p "${dockerdir}"
- pushd "${PATCH_DIR}/test-patch-docker" >/dev/null || return 1
+ if [[ ${lines} -gt 0 ]]; then
+ if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \
+ [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 38 ]]; then
+
+ # versions less than 18 don't support having the
+ # Dockerfile outside of the build context. Let's fall back to
+ # pre-YETUS-723 behavior and put the re-written Dockerfile
+ # outside of the source tree rather than go through a lot of
+ # machinations. This means COPY, ADD, etc do not work, but
+ # whatever
+
+ popd >/dev/null || return 1
+ buildfile="${PATCH_DIR}/test-patch-docker/Dockerfile"
+ dockerdir="${PATCH_DIR}/test-patch-docker"
+ mkdir -p "${dockerdir}"
+ pushd "${PATCH_DIR}/test-patch-docker" >/dev/null || return 1
+ fi
fi
(
@@ -700,10 +704,13 @@ function docker_run_image
fi
) > "${buildfile}"
- if [[ "${DOCKER_VERSION[0]}" -lt 18 ]] && [[ ${lines} -gt 0 ]]; then
- # Need to put our re-constructed Dockerfile in a place
- # where it can be referenced in the output post-run
- cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile"
+ if [[ ${lines} -gt 0 ]]; then
+ if [[ "${DOCKER_VERSION[0]}" -lt 1 ]] || \
+ [[ "${DOCKER_VERSION[0]}" -lt 2 && "${DOCKER_VERSION[1]}" -lt 38 ]]; then
+ # Need to put our re-constructed Dockerfile in a place
+ # where it can be referenced in the output post-run
+ cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile"
+ fi
fi
if [[ -n "${DOCKER_CACHE_FROM}" ]]; then
@@ -799,7 +806,7 @@ function docker_run_image
client=$(docker_version Client)
server=$(docker_version Server)
- dockerversion="Client=${client} Server=${server}"
+ dockerversion="ClientAPI=${client} ServerAPI=${server}"
# make the kernel prefer to kill us if we run out of RAM
DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
diff --git a/precommit/src/main/shell/robots.d/githubactions.sh b/precommit/src/main/shell/robots.d/githubactions.sh
new file mode 100755
index 0000000..f59eb85
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/githubactions.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+if [[ "${GITHUB_ACTIONS}" == true ]] &&
+ declare -f compile_cycle >/dev/null; then
+ # shellcheck disable=SC2034
+ ROBOT=true
+ # shellcheck disable=SC2034
+ PATCH_DIR=${PATCH_DIR:-${GITHUB_WORKSPACE}/yetus}
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE="GHSHA:${GITHUB_SHA}"
+
+ # shellcheck disable=SC2034
+ INSTANCE=${GITHUB_RUN_NUMBER}
+
+ # shellcheck disable=SC2034
+ ROBOTTYPE=githubactions
+
+ # shellcheck disable=SC2034
+ GITHUB_REPO="${GITHUB_REPOSITORY}"
+
+ if [[ "${GITHUB_EVENT_NAME}" == push ]]; then
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=""
+ #shellcheck disable=SC2034,SC2153
+ PATCH_BRANCH=$(echo "${GITHUB_REF}" | cut -f3- -d/)
+ # shellcheck disable=SC2034
+ BUILDMODE=full
+ add_docker_env BUILDMODE
+ elif [[ "${GITHUB_EVENT_NAME}" == pull_request ]]; then
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=$(echo "${GITHUB_REF}" | cut -f3 -d/)
+ PATCH_OR_ISSUE="GH:${PATCH_OR_ISSUE}"
+ # shellcheck disable=SC2034
+ PATCH_BRANCH=${GITHUB_BASE_REF}
+ fi
+
+ add_docker_env \
+ GITHUB_WORKSPACE \
+ GITHUB_SHA \
+ GITHUB_REPOSITORY \
+ GITHUB_RUN_NUMBER \
+ GITHUB_REF \
+ GITHUB_ACTIONS \
+ GITHUB_EVENT_NAME \
+ GITHUB_BASE_REF
+
+ # shellcheck disable=SC2034
+ BUILD_URL_CONSOLE=console
+
+ # shellcheck disable=SC2034
+ CONSOLE_USE_BUILD_URL=true
+
+ if [[ -d ${BASEDIR}/.git ]]; then
+ echo "Updating the local git repo to include all branches/tags:"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
+ "${GIT}" fetch --tags
+ popd >/dev/null || exit 1
+ fi
+
+ yetus_add_array_element EXEC_MODES GitHubActions
+fi
+
+function githubactions_set_plugin_defaults
+{
+ # shellcheck disable=SC2034
+ GITHUB_REPO="${GITHUB_REPOSITORY}"
+}
+
diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh
index 32b5456..921368d 100755
--- a/precommit/src/main/shell/test-patch.d/github.sh
+++ b/precommit/src/main/shell/test-patch.d/github.sh
@@ -414,22 +414,33 @@ function github_locate_sha_patch
number=$("${GREP}" number "${PATCH_DIR}/github-search.json" | \
head -1 | \
"${AWK}" '{print $NF}')
- number=${number//\s/}
+ number=${number//\s}
number=${number%,}
# Semaphore CI doesn't tell us if the sha is a PR or not, so...
- if [[ -z "${number}" ]] && [[ "${ROBOTTYPE}" = semaphoreci ]]; then
+ if [[ -z "${number}" ]]; then
+ if [[ "${ROBOTTYPE}" = semaphoreci ]]; then
- echo "This appears to be a full build on Semaphore CI. Switching modes."
+ echo "This appears to be a full build on Semaphore CI. Switching modes."
- PATCH_BRANCH=${SEMAPHORE_GIT_BRANCH}
+ PATCH_BRANCH=${SEMAPHORE_GIT_BRANCH}
- # shellcheck disable=SC2034
- PATCH_OR_ISSUE=""
- # shellcheck disable=SC2034
- BUILDMODE=full
- set_buildmode
- return 0
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=""
+ # shellcheck disable=SC2034
+ BUILDMODE=full
+ add_docker_env BUILDMODE
+ set_buildmode
+ return 0
+ else
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=""
+ # shellcheck disable=SC2034
+ BUILDMODE=full
+ add_docker_env BUILDMODE
+ set_buildmode
+ return 0
+ fi
fi
github_locate_pr_patch "GH:${number}" "${patchout}" "${diffout}"
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index 58d011d..46cef73 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -972,6 +972,7 @@ function parse_args
if [[ -z "${PATCH_OR_ISSUE}"
&& "${BUILDMODE}" = patch ]]; then
+ yetus_error "ERROR: No patch given."
yetus_usage
exit 1
fi