You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ms...@apache.org on 2022/12/22 13:11:34 UTC
[openwhisk] branch master updated: fixes to use different secrets, show the results in the log (#5367)
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 21c9a6363 fixes to use different secrets, show the results in the log (#5367)
21c9a6363 is described below
commit 21c9a6363fe14e581ec67da281e1762db02ae557
Author: Michele Sciabarra <mi...@nuvolaris.io>
AuthorDate: Thu Dec 22 14:11:26 2022 +0100
fixes to use different secrets, show the results in the log (#5367)
* fixes to use different secrets, show the results in the log
* propagating aws variabales and disabiling a test if the key is there ut empty
Co-authored-by: Michele Sciabarra <mi...@sciabarra.com>
---
.github/workflows/0-on-demand.yaml | 10 +++++-----
.github/workflows/1-unit.yaml | 9 +++++++--
.github/workflows/2-system.yaml | 10 +++++-----
.github/workflows/3-multi-runtime.yaml | 10 +++++-----
.github/workflows/4-standalone.yaml | 10 +++++-----
.github/workflows/5-scheduler.yaml | 10 +++++-----
.github/workflows/6-performance.yaml | 8 ++++----
.github/workflows/README.md | 18 +++++++++---------
.../org/apache/openwhisk/core/database/s3/S3Aws.scala | 2 +-
tools/github/checkAndUploadLogs.sh | 10 +++++++---
tools/github/s3-upload.sh | 14 +++++++-------
tools/github/setup.sh | 3 +++
12 files changed, 63 insertions(+), 51 deletions(-)
diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml
index 5d8d66ea3..a39a379d9 100644
--- a/.github/workflows/0-on-demand.yaml
+++ b/.github/workflows/0-on-demand.yaml
@@ -47,10 +47,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
@@ -74,7 +74,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml
index 144720625..c94061250 100644
--- a/.github/workflows/1-unit.yaml
+++ b/.github/workflows/1-unit.yaml
@@ -34,7 +34,12 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
+
+ # some tests need also this even if they are empty on pull_requests...
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
@@ -56,7 +61,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml
index 9295f921f..79c5ea6eb 100644
--- a/.github/workflows/2-system.yaml
+++ b/.github/workflows/2-system.yaml
@@ -34,10 +34,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
@@ -56,7 +56,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml
index 6bdcf7703..e74bba246 100644
--- a/.github/workflows/3-multi-runtime.yaml
+++ b/.github/workflows/3-multi-runtime.yaml
@@ -34,10 +34,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
@@ -56,7 +56,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml
index 92a9a7d82..15a1c8ff3 100644
--- a/.github/workflows/4-standalone.yaml
+++ b/.github/workflows/4-standalone.yaml
@@ -34,10 +34,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
@@ -56,7 +56,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml
index cb9234f20..02b526d55 100644
--- a/.github/workflows/5-scheduler.yaml
+++ b/.github/workflows/5-scheduler.yaml
@@ -34,10 +34,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
@@ -56,7 +56,7 @@ jobs:
run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
continue-on-error: true
- id: logs
- name: Upload Logs
+ name: Show results and Upload logs
run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
- name: Slack Notification
run: >
diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml
index cde8c5fbf..55e811b1d 100644
--- a/.github/workflows/6-performance.yaml
+++ b/.github/workflows/6-performance.yaml
@@ -34,10 +34,10 @@ env:
SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
# (optional) s3 log upload
- AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: ${{ secrets.AWS_REGION }}
+ LOG_BUCKET: ${{ secrets.LOG_BUCKET }}
+ LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }}
+ LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }}
+ LOG_REGION: ${{ secrets.LOG_REGION }}
# github
GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index 55473d04e..dbe6c7089 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -35,19 +35,19 @@ The build uploads the logs to an s3 bucket allowing to inspect them with a brows
You need to create the bucket with the following commands:
```
-AWS_BUCKET=<name-of-your-bucket>
-AWS_REGION=<the-region-you-use>
-aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION
-aws s3 website s3://$AWS_BUCKET/ --index-document index.html
-aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET
+LOG_BUCKET=<name-of-your-bucket>
+LOG_REGION=<the-region-you-use>
+aws s3 mb s3://$LOG_BUCKET --region $LOG_REGION
+aws s3 website s3://$LOG_BUCKET/ --index-document index.html
+aws s3api put-bucket-acl --acl public-read --bucket $LOG_BUCKET
```
To enable upload to the created bucket you need to set the following secrets:
-- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before.
-- `AWS_ACCESS_KEY_ID`: your aws access key.
-- `AWS_SECRET_ACCESS_KEY`: your aws secret key.
-- `AWS_REGION`: important: the region where your bucket is.
+- `LOG_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before.
+- `LOG_ACCESS_KEY_ID`: your aws access key.
+- `LOG_SECRET_ACCESS_KEY`: your aws secret key.
+- `LOG_REGION`: important: the region where your bucket is.
## Slack notification
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala
index 722997ac9..952598d19 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala
@@ -57,7 +57,7 @@ trait S3Aws extends FlatSpec {
override protected def withFixture(test: NoArgTest) = {
assume(
- secretAccessKey != null,
+ secretAccessKey != null && secretAccessKey != "",
"'AWS_SECRET_ACCESS_KEY' env not configured. Configure following " +
"env variables for test to run. 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_REGION'")
diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh
index 79f169720..da96610a7 100755
--- a/tools/github/checkAndUploadLogs.sh
+++ b/tools/github/checkAndUploadLogs.sh
@@ -16,11 +16,15 @@
# limitations under the License.
#
+# showing test results on the CI log
+INDEX="tests/build/reports/tests/testCoverageLean/index.html"
+test -f "$INDEX" && lynx -dump file://$PWD/$INDEX | grep .
+
# check variables
-for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION
+for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY LOG_REGION
do
if test -z "${!i}"
- then echo "Required Environment Variable Missing: $i" ; exit 1
+ then echo "Required Environment Variable Missing: $i" ; exit 0
fi
done
@@ -42,7 +46,7 @@ TAGS=""
[[ "$2" == "Unit" ]] && TAGS="db"
LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}"
-BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com"
+BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com"
echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/"
echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html"
diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh
index 38c685a04..5d2d0141b 100755
--- a/tools/github/s3-upload.sh
+++ b/tools/github/s3-upload.sh
@@ -17,7 +17,7 @@
#
# check variables
-for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY
do
if test -z "${!i}"
then echo "Please set $i" ; exit 1
@@ -32,7 +32,7 @@ FROM="$1"
TO="$2"
BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html"
-BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/"
+BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com/"
# install rclone
if ! which rclone
@@ -41,13 +41,13 @@ fi
RCLONE="rclone --config /dev/null \
--s3-provider AWS \
- --s3-region $AWS_REGION \
+ --s3-region $LOG_REGION \
--s3-acl public-read \
- --s3-access-key-id $AWS_ACCESS_KEY_ID \
- --s3-secret-access-key $AWS_SECRET_ACCESS_KEY"
+ --s3-access-key-id $LOG_ACCESS_KEY_ID \
+ --s3-secret-access-key $LOG_SECRET_ACCESS_KEY"
curl -s "$BROWSER" |\
sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\
- $RCLONE rcat ":s3:$AWS_BUCKET/index.html"
+ $RCLONE rcat ":s3:$LOG_BUCKET/index.html"
-$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/"
+$RCLONE copyto "$FROM" ":s3:$LOG_BUCKET/$TO/"
diff --git a/tools/github/setup.sh b/tools/github/setup.sh
index e72b04605..aefb21047 100755
--- a/tools/github/setup.sh
+++ b/tools/github/setup.sh
@@ -34,6 +34,9 @@ function retry() {
fi
}
+# lynx utility to show test results on the job run
+sudo apt-get -y install lynx
+
# setup docker to listen in port 4243
sudo systemctl stop docker
sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service