You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2021/04/08 12:16:20 UTC
[ozone] branch master updated: HDDS-4969. Extract check dependency
installation from Github Actions workflow (#2029)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4f76d03 HDDS-4969. Extract check dependency installation from Github Actions workflow (#2029)
4f76d03 is described below
commit 4f76d03a727327a7343e07b807673b8cb01bb5c7
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Thu Apr 8 14:16:00 2021 +0200
HDDS-4969. Extract check dependency installation from Github Actions workflow (#2029)
---
.github/workflows/post-commit.yml | 23 ------
.gitignore | 2 +
hadoop-ozone/dev-support/checks/_lib.sh | 106 ++++++++++++++++++++++++++
hadoop-ozone/dev-support/checks/acceptance.sh | 5 ++
hadoop-ozone/dev-support/checks/bats.sh | 5 ++
hadoop-ozone/dev-support/checks/kubernetes.sh | 16 ++++
6 files changed, 134 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index c869a49..b34a6c8 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -72,12 +72,6 @@ jobs:
steps:
- name: Checkout project
uses: actions/checkout@v2
- - name: Install bats
- run: |
- cd /tmp
- curl -LSs https://github.com/bats-core/bats-core/archive/v1.2.1.tar.gz | tar xzf -
- cd bats-core-1.2.1
- sudo ./install.sh /usr/local
- name: Execute tests
run: ./hadoop-ozone/dev-support/checks/${{ github.job }}.sh
- name: Summary of failures
@@ -217,8 +211,6 @@ jobs:
mkdir -p /mnt/ozone/hadoop-ozone/dist/target
tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
sudo chmod -R a+rwX /mnt/ozone/hadoop-ozone/dist/target
- - name: Install robotframework
- run: sudo pip install robotframework
- name: Execute tests
run: |
cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
@@ -322,21 +314,6 @@ jobs:
run: |
mkdir -p /mnt/ozone/hadoop-ozone/dist/target
tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
- - name: Install robotframework
- run: sudo pip install robotframework
- - name: Install k3s
- run: curl -sfL https://get.k3s.io | sh -
- - name: Copy Kubernetes config file
- run: |
- sudo mkdir ~/.kube
- sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
- sudo chown $(id -u) ~/.kube/config
- - name: Install flekszible
- run: |
- cd /tmp
- wget https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz -O - | tar -zx
- chmod +x flekszible
- sudo mv flekszible /usr/bin/flekszible
- name: Execute tests
run: |
cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
diff --git a/.gitignore b/.gitignore
index e09c2eb..d3a3e1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,3 +65,5 @@ hadoop-hdds/docs/public
hadoop-ozone/recon/node_modules
.mvn
+
+.dev-tools
diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh
new file mode 100644
index 0000000..4e6b198
--- /dev/null
+++ b/hadoop-ozone/dev-support/checks/_lib.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+check_name="$(basename "${BASH_SOURCE[1]}")"
+check_name="${check_name%.sh}"
+
+: ${TOOLS_DIR:=$(pwd)/.dev-tools} # directory for tools
+: ${OZONE_PREFER_LOCAL_TOOL:=true} # skip install if tools are already available (eg. via package manager)
+
+## @description Install a dependency. Only first argument is mandatory.
+## @param name of the tool
+## @param the directory for binaries, relative to the tool directory; added to PATH.
+## @param the directory for the tool, relative to TOOLS_DIR
+## @param name of the executable, for testing if it is already installed
+## @param name of the function that performs actual installation steps
+_install_tool() {
+ local tool bindir dir bin func
+
+ tool="$1"
+ bindir="${2:-}"
+ dir="${TOOLS_DIR}"/"${3:-"${tool}"}"
+ bin="${4:-"${tool}"}"
+ func="${5:-"_install_${tool}"}"
+
+ if [[ "${OZONE_PREFER_LOCAL_TOOL}" == "true" ]] && which "$bin" >& /dev/null; then
+ echo "Skip installing $bin, as it's already available on PATH."
+ return
+ fi
+
+ if [[ ! -d "${dir}" ]]; then
+ mkdir -pv "${dir}"
+ pushd "${dir}"
+ eval "${func}"
+ echo "Installed ${tool} in ${dir}"
+ popd
+ fi
+
+ if [[ -n "${bindir}" ]]; then
+ bindir="${dir}"/"${bindir}"
+ if [[ -d "${bindir}" ]]; then
+ if [[ "${OZONE_PREFER_LOCAL_TOOL}" == "true" ]]; then
+ export PATH="${PATH}:${bindir}"
+ else
+ export PATH="${bindir}:${PATH}"
+ fi
+ fi
+ fi
+}
+
+install_bats() {
+ _install_tool bats bats-core-1.2.1/bin
+}
+
+_install_bats() {
+ curl -LSs https://github.com/bats-core/bats-core/archive/v1.2.1.tar.gz | tar -xz -f -
+}
+
+install_k3s() {
+ _install_tool k3s
+}
+
+_install_k3s() {
+ curl -sfL https://get.k3s.io | sh -
+ sudo chmod a+r $KUBECONFIG
+}
+
+install_flekszible() {
+ _install_tool flekszible bin
+}
+
+_install_flekszible() {
+ mkdir bin
+ curl -LSs https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz | tar -xz -f - -C bin
+ chmod +x bin/flekszible
+}
+
+install_virtualenv() {
+ _install_tool virtualenv
+}
+
+_install_virtualenv() {
+ sudo pip3 install virtualenv
+}
+
+install_robot() {
+ _install_tool robot venv/bin
+}
+
+_install_robot() {
+ virtualenv venv
+ source venv/bin/activate
+ pip install robotframework
+}
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index a96aff7..d271f12 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -16,6 +16,11 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+source "${DIR}/_lib.sh"
+
+install_virtualenv
+install_robot
+
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"}
OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout)
diff --git a/hadoop-ozone/dev-support/checks/bats.sh b/hadoop-ozone/dev-support/checks/bats.sh
index 2e1bbad..f5b2cc9 100755
--- a/hadoop-ozone/dev-support/checks/bats.sh
+++ b/hadoop-ozone/dev-support/checks/bats.sh
@@ -13,9 +13,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "${DIR}/../../.." || exit 1
+source "${DIR}/_lib.sh"
+
+install_bats
+
REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/bats"}
mkdir -p "${REPORT_DIR}"
REPORT_FILE="${REPORT_DIR}/summary.txt"
diff --git a/hadoop-ozone/dev-support/checks/kubernetes.sh b/hadoop-ozone/dev-support/checks/kubernetes.sh
index ea66313..70105ee 100755
--- a/hadoop-ozone/dev-support/checks/kubernetes.sh
+++ b/hadoop-ozone/dev-support/checks/kubernetes.sh
@@ -13,9 +13,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+: ${KUBECONFIG:=/etc/rancher/k3s/k3s.yaml}
+
+export KUBECONFIG
+
+source "${DIR}/_lib.sh"
+
+install_flekszible
+install_virtualenv
+install_robot
+if [[ "$(uname -s)" = "Darwin" ]]; then
+ echo "Skip installing k3s, not supported on Mac. Make sure a working Kubernetes cluster is available." >&2
+else
+ install_k3s
+fi
+
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/kubernetes"}
OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org