You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ms...@apache.org on 2022/12/16 15:33:39 UTC
[openwhisk] branch master updated: GitHub action (#5361)
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new f717619c8 GitHub action (#5361)
f717619c8 is described below
commit f717619c8c25b54294cfa195732732e85c5d9431
Author: Michele Sciabarra <mi...@nuvolaris.io>
AuthorDate: Fri Dec 16 16:33:32 2022 +0100
GitHub action (#5361)
* github action initial commit
* fix blank
* debug
* fix push
* fix java
* add setup
* triggering build
* run unit test
* ooops
* updating debugAction
* waitifdebug
* fix cleanup
* final test
* disable ngrok
* reworked the PR - multiple jobs and slack
* split in multiple build
* github actions updated
* scancode fixes
* forcing a corresponding version of jdk in ci and docker image and renaming S3_LOG_BUCKET to AWS_BUCKET to fix tests
* more fixes - only one test failing against 1.0.0
* finally passing all the tests on 1.0.0
* restored master variables
* restored more master files
* triggered on push and pull request
* unit test too
* wskadmin fixes
* improve message
* dummy enabled
* better git message
* cleanup the log messages
* unnecessary change to master - even if beneficial
* improved messages (and a test disabled)
* reverting commented test
* get rid of DummyKO no more needed
Co-authored-by: Michele Sciabarra <mi...@sciabarra.com>
---
.github/workflows/0-on-demand.yaml | 92 ++++++++++++++++++++++
.github/workflows/1-unit.yaml | 70 ++++++++++++++++
.github/workflows/2-system.yaml | 70 ++++++++++++++++
.github/workflows/3-multi-runtime.yaml | 70 ++++++++++++++++
.github/workflows/4-standalone.yaml | 70 ++++++++++++++++
.github/workflows/5-scheduler.yaml | 70 ++++++++++++++++
.github/workflows/6-performance.yaml | 84 ++++++++++++++++++++
.github/workflows/README.md | 56 +++++++++++++
.gitignore | 4 +
common/scala/Dockerfile | 3 +-
tools/github/checkAndUploadLogs.sh | 59 ++++++++++++++
tools/github/debugAction.sh | 69 ++++++++++++++++
tools/github/flake8.sh | 42 ++++++++++
.../Dockerfile => tools/github/runDummyTests.sh | 21 ++---
.../github/runLeanSystemTests.sh | 27 ++++---
.../github/runMultiRuntimeTests.sh | 27 ++++---
.../github/runSchedulerTests.sh | 26 +++---
tools/github/runStandaloneTests.sh | 64 +++++++++++++++
.../Dockerfile => tools/github/runSystemTests.sh | 27 ++++---
.../Dockerfile => tools/github/runUnitTests.sh | 25 +++---
tools/github/s3-upload.sh | 53 +++++++++++++
tools/github/scan.sh | 52 ++++++++++++
tools/github/setup.sh | 79 +++++++++++++++++++
.../Dockerfile => tools/github/waitIfDebug.sh | 32 ++++----
.../Dockerfile => tools/github/writeOnSlack.sh | 23 +++---
25 files changed, 1112 insertions(+), 103 deletions(-)
diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml
new file mode 100644
index 000000000..5d8d66ea3
--- /dev/null
+++ b/.github/workflows/0-on-demand.yaml
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: On Demand Tests
+
+on:
+ workflow_dispatch:
+ inputs:
+ enable_ngrok_debug:
+ description: "Enable Ngrok Debugging"
+ required: true
+ type: boolean
+ default: false
+ test_suite:
+ description: Select Test Suite to run
+ type: choice
+ options:
+ - Unit
+ - System
+ - MultiRuntime
+ - Standalone
+ - Scheduler
+ - Performance
+ - Dummy
+
+env:
+ # openwhisk env
+ TEST_SUITE: ${{ inputs.test_suite }}
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+ # (optional) you need to add as secrets an ngrok token and a password to debug a build on demand
+ NGROK_DEBUG: ${{ inputs.enable_ngrok_debug }}
+ NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }}
+ NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Debug Action (if requested)
+ run: ./tools/github/debugAction.sh
+ - name: Wait for Debug (if requested)
+ run: ./tools/github/waitIfDebug.sh
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml
new file mode 100644
index 000000000..144720625
--- /dev/null
+++ b/.github/workflows/1-unit.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Unit Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: Unit
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml
new file mode 100644
index 000000000..9295f921f
--- /dev/null
+++ b/.github/workflows/2-system.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: System Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: System
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml
new file mode 100644
index 000000000..6bdcf7703
--- /dev/null
+++ b/.github/workflows/3-multi-runtime.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: MultiRuntime Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: MultiRuntime
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml
new file mode 100644
index 000000000..92a9a7d82
--- /dev/null
+++ b/.github/workflows/4-standalone.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Standalone Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: Standalone
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml
new file mode 100644
index 000000000..cb9234f20
--- /dev/null
+++ b/.github/workflows/5-scheduler.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Scheduler Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: Scheduler
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - id: tests
+ name: Run Tests
+ run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh"
+ continue-on-error: true
+ - id: logs
+ name: Upload Logs
+ run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }}
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nlogs:' ${{ steps.logs.outputs.logs }}
+ $'\nreport:' ${{ steps.logs.outputs.report }}
+ - name: Results
+ run: test "${{ steps.tests.outcome }}" == "success"
diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml
new file mode 100644
index 000000000..cde8c5fbf
--- /dev/null
+++ b/.github/workflows/6-performance.yaml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Performance Tests
+
+on:
+ # build on push
+ push:
+ # build on pull requests
+ pull_request:
+
+env:
+ # openwhisk env
+ TEST_SUITE: Performance
+ ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing"
+ GRADLE_PROJS_SKIP: ""
+
+ ## secrets
+ # (optional) slack incoming wehbook for notifications
+ SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}}
+
+ # (optional) s3 log upload
+ AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: ${{ secrets.AWS_REGION }}
+
+ # github
+ GH_BUILD: ${{ github.event_name }}-${{ github.sha }}
+ GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+ openwhisk:
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: "Setup"
+ run: ./tools/github/setup.sh
+ - run: ./tests/performance/preparation/deploy.sh
+ - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m
+ continue-on-error: true
+ - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m
+ continue-on-error: true
+ - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m
+ continue-on-error: true
+ - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 4 1 2 2m
+ continue-on-error: true
+ - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 100 110 2 2m
+ continue-on-error: true
+ - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 100 110 2 2m
+ continue-on-error: true
+ - run: OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ApiV1Simulation
+ continue-on-error: true
+ - run: OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" PAUSE_BETWEEN_INVOKES="100" ./gradlew gatlingRun-org.apache.openwhisk.LatencySimulation
+ continue-on-error: true
+ - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation
+ continue-on-error: true
+ - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ASYNC="true" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation
+ continue-on-error: true
+ # The following configuration does not make much sense. But we do not have enough users. But it's good to verify, that the test is still working.
+ - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation
+ continue-on-error: true
+ - name: Slack Notification
+ run: >
+ ./tools/github/writeOnSlack.sh
+ "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD}
+ $'\nbranch:' $GH_BRANCH
+ $'\nmessage:' "$(git log -1 --oneline | cat)"
+ $'\nCheck GitHub logs for results'
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
new file mode 100644
index 000000000..55473d04e
--- /dev/null
+++ b/.github/workflows/README.md
@@ -0,0 +1,56 @@
+# How to use those workflows
+
+There are a few [GitHub secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to configure to fully leverage the build.
+
+You can use and set the followings secrets also in your fork.
+
+## Ngrok Debugging
+
+You can debug a GitHub Action build using [NGROK](https://ngrok.com/).
+
+It is disabled for automated build triggered by push and pull_requests.
+
+You can trigger a workflow run manually enabling ngrok debugging.
+
+It will open an ssh connection to the VM and keep it up and running for one hour.
+The connection url is showns in the log for debugAction.sh
+
+You can then connect to the build vm, and debug it.
+You need to use a password of your choice to access it.
+
+You can continue the build with `touch /tmp/continue`.
+You can abort the build with `touch /tmp/abort`.
+
+To enable this option you have to register to Ngrok, using the fee account and get the NGROK Token.
+
+Then set the following secrets:
+
+- `NGROK_TOKEN` to the ngrok token.
+- `NGROK_PASSWORD` to a password of choice to access the build with the ssh command generated.
+
+## Log Upload
+
+The build uploads the logs to an s3 bucket allowing to inspect them with a browser.
+
+You need to create the bucket with the following commands:
+
+```
+AWS_BUCKET=<name-of-your-bucket>
+AWS_REGION=<the-region-you-use>
+aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION
+aws s3 website s3://$AWS_BUCKET/ --index-document index.html
+aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET
+```
+
+To enable upload to the created bucket you need to set the following secrets:
+
+- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before.
+- `AWS_ACCESS_KEY_ID`: your aws access key.
+- `AWS_SECRET_ACCESS_KEY`: your aws secret key.
+- `AWS_REGION`: important: the region where your bucket is.
+
+## Slack notification
+
+If you want to get notified of what happens on slack, create an [Incoming Web Hook](https://api.slack.com/messaging/webhooks) and then set the following secret:
+
+- `SLACK_WEBHOOK`: the incoming webhook url provided by slack.
diff --git a/.gitignore b/.gitignore
index 284e8d225..7a15a629f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -96,3 +96,7 @@ core/routemgmt/deleteApi/utils.js
core/routemgmt/getApi/apigw-utils.js
core/routemgmt/getApi/package-lock.json
core/routemgmt/getApi/utils.js
+
+# vscode metals
+.bloop/
+.metals/
diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile
index 6db867ff2..c7ebd2b9e 100644
--- a/common/scala/Dockerfile
+++ b/common/scala/Dockerfile
@@ -14,7 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
+# if you change version of openjsk, also update tools/github/setup.sh to download the corresponding jdk
+FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.12_7_openj9-0.27.0
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh
new file mode 100755
index 000000000..79f169720
--- /dev/null
+++ b/tools/github/checkAndUploadLogs.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# check variables
+for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION
+do
+ if test -z "${!i}"
+ then echo "Required Environment Variable Missing: $i" ; exit 1
+ fi
+done
+
+# Disable abort script at first error as we require the logs to be uploaded
+# even if check and log collection fails
+# set -e
+
+
+SECONDS=0
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR
+
+LOG_NAME="$1"
+
+# tags is db only when the test is unit
+TAGS=""
+[[ "$2" == "Unit" ]] && TAGS="db"
+
+LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}"
+BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com"
+
+echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/"
+echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html"
+
+echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}/" >>${GITHUB_OUTPUT:-/dev/stdin}
+echo "report=${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" >>${GITHUB_OUTPUT:-/dev/stdin}
+
+ansible-playbook -i ansible/environments/local ansible/logs.yml
+
+./tools/build/checkLogs.py logs "$TAGS"
+
+./tools/github/s3-upload.sh "$PWD/logs" "$LOG_DIR"
+
+echo "Time taken for ${0##*/} is $SECONDS secs"
diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh
new file mode 100755
index 000000000..6ebe6e243
--- /dev/null
+++ b/tools/github/debugAction.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [[ -z "$NGROK_DEBUG" ]] || [[ "$NGROK_DEBUG" == "false" ]]
+then exit 0
+fi
+
+if [[ -z "$NGROK_TOKEN" ]]
+then echo "Please set 'NGROK_TOKEN'"
+ exit 1
+fi
+
+if [[ -z "$NGROK_PASSWORD" ]]
+then echo "Please set 'NGROK_PASSWORD'"
+ exit 1
+fi
+
+echo "### Install ngrok ###"
+if ! test -e ./ngrok
+then
+ wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-386.zip
+ unzip ngrok-stable-linux-386.zip
+ chmod +x ./ngrok
+fi
+
+echo "### Update user: $USER password ###"
+echo -e "$NGROK_PASSWORD\n$NGROK_PASSWORD" | sudo passwd "$USER"
+
+echo "### Start ngrok proxy for 22 port ###"
+
+rm -f .ngrok.log
+./ngrok authtoken "$NGROK_TOKEN"
+./ngrok tcp 22 --log ".ngrok.log" &
+
+sleep 10
+HAS_ERRORS=$(grep "command failed" < .ngrok.log)
+
+if [[ -z "$HAS_ERRORS" ]]; then
+ MSG="To connect: $(grep -o -E "tcp://(.+)" < .ngrok.log | sed "s/tcp:\/\//ssh $USER@/" | sed "s/:/ -p /")"
+ echo ""
+ echo "=========================================="
+ echo "$MSG"
+ echo "=========================================="
+ if test -n "$SLACK_WEBHOOK"
+ then
+ echo -n '{"text":' >/tmp/msg$$
+ echo -n "$MSG" | jq -Rsa . >>/tmp/msg$$
+ echo -n '}' >>/tmp/msg$$
+ curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK"
+ fi
+else
+ echo "$HAS_ERRORS"
+ exit 1
+fi
diff --git a/tools/github/flake8.sh b/tools/github/flake8.sh
new file mode 100755
index 000000000..98b7c0d9c
--- /dev/null
+++ b/tools/github/flake8.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pip3 install --user --upgrade flake8
+
+# These files do not have a .py extension so flake8 will not scan them
+declare -a PYTHON_FILES=("."
+ "./tools/admin/wskadmin"
+ "./tools/build/citool"
+ "./tools/build/redo")
+
+echo 'Flake8: first round (fast fail) stops the build if there are any Python 3 syntax errors...'
+for i in "${PYTHON_FILES[@]}"
+do
+ flake8 "$i" --select=E999,F821 --statistics
+ RETURN_CODE=$?
+ if [ $RETURN_CODE != 0 ]; then
+ echo 'Flake8 found Python 3 syntax errors above. See: https://docs.python.org/3/howto/pyporting.html'
+ exit $RETURN_CODE
+ fi
+done
+
+echo 'Flake8: second round to find any other stylistic issues...'
+for i in "${PYTHON_FILES[@]}"
+do
+ flake8 "$i" --ignore=E,W503,W504,W605 --max-line-length=127 --statistics
+done
diff --git a/common/scala/Dockerfile b/tools/github/runDummyTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runDummyTests.sh
index 6db867ff2..11564531e
--- a/common/scala/Dockerfile
+++ b/tools/github/runDummyTests.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
-RUN mkdir /logs
-
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
-
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+mkdir $ROOTDIR/logs
+echo "<h1>$(date)</h1>" >$ROOTDIR/logs/now.html
+exit 0
diff --git a/common/scala/Dockerfile b/tools/github/runLeanSystemTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runLeanSystemTests.sh
index 6db867ff2..372fbc1e4
--- a/common/scala/Dockerfile
+++ b/tools/github/runLeanSystemTests.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +16,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR/tools/travis
+
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_LEAN_SYSTEM"
+export GRADLE_COVERAGE=true
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+./setupPrereq.sh /ansible/files/runtimes-nodeonly.json
-RUN mkdir /logs
+./distDocker-lean.sh
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
+./setupLeanSystem.sh /ansible/files/runtimes-nodeonly.json
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+./runTests.sh
diff --git a/common/scala/Dockerfile b/tools/github/runMultiRuntimeTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runMultiRuntimeTests.sh
index 6db867ff2..e2de0c59b
--- a/common/scala/Dockerfile
+++ b/tools/github/runMultiRuntimeTests.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +16,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR/tools/travis
+
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_MULTI_RUNTIME"
+export GRADLE_COVERAGE=true
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+./setupPrereq.sh
-RUN mkdir /logs
+./distDocker.sh
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
+./setupSystem.sh
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+./runTests.sh
diff --git a/common/scala/Dockerfile b/tools/github/runSchedulerTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runSchedulerTests.sh
index 6db867ff2..000de8080
--- a/common/scala/Dockerfile
+++ b/tools/github/runSchedulerTests.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +16,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR/tools/travis
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SCHEDULER"
+export GRADLE_COVERAGE=true
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+./setupPrereq.sh /ansible/files/runtimes-nodeonly.json
-RUN mkdir /logs
+./distDocker.sh
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
+./setupSystem.sh /ansible/files/runtimes-nodeonly.json
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+./runTests.sh
diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh
new file mode 100755
index 000000000..88dac0762
--- /dev/null
+++ b/tools/github/runStandaloneTests.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_STANDALONE"
+export GRADLE_COVERAGE=true
+
+cd $ROOTDIR/ansible
+$ANSIBLE_CMD setup.yml
+$ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.json"
+$ANSIBLE_CMD downloadcli-github.yml
+
+# Install kubectl
+curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.25.3/bin/linux/amd64/kubectl
+chmod +x kubectl
+sudo cp kubectl /usr/local/bin/kubectl
+
+# Install kind
+curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.17.0/kind-linux-amd64
+chmod +x kind
+sudo cp kind /usr/local/bin/kind
+
+kind create cluster --wait 5m
+export KUBECONFIG="$(kind get kubeconfig-path)"
+kubectl config set-context --current --namespace=default
+
+# This is required because it is timed out to pull the image during the test.
+docker pull openwhisk/action-nodejs-v14:nightly
+docker pull openwhisk/dockerskeleton:nightly
+docker pull openwhisk/example:nightly
+docker pull openwhisk/apigateway:0.11.0
+
+cd $ROOTDIR
+TERM=dumb ./gradlew :core:standalone:build \
+ :core:monitoring:user-events:distDocker
+
+cd $ROOTDIR
+TERM=dumb ./gradlew :core:standalone:cleanTest \
+ :core:standalone:test \
+ :core:monitoring:user-events:reportTestScoverage
+
+# Run test in end as it publishes the coverage also
+cd $ROOTDIR/tools/travis
+./runTests.sh
diff --git a/common/scala/Dockerfile b/tools/github/runSystemTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runSystemTests.sh
index 6db867ff2..8325caddb
--- a/common/scala/Dockerfile
+++ b/tools/github/runSystemTests.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +16,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR/tools/travis
+
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SYSTEM"
+export GRADLE_COVERAGE=true
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+./setupPrereq.sh /ansible/files/runtimes-nodeonly.json
-RUN mkdir /logs
+./distDocker.sh
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
+./setupSystem.sh /ansible/files/runtimes-nodeonly.json
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+./runTests.sh
diff --git a/common/scala/Dockerfile b/tools/github/runUnitTests.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/runUnitTests.sh
index 6db867ff2..b10418c9e
--- a/common/scala/Dockerfile
+++ b/tools/github/runUnitTests.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+set -e
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
-RUN mkdir /logs
+cd $ROOTDIR/tools/travis
+export TESTCONTAINERS_RYUK_DISABLED="true"
+export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB"
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
+./setupPrereq.sh
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+cat "$ROOTDIR/tests/src/test/resources/application.conf"
+
+./distDocker.sh
+
+./runTests.sh
diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh
new file mode 100755
index 000000000..38c685a04
--- /dev/null
+++ b/tools/github/s3-upload.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# check variables
+for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+do
+ if test -z "${!i}"
+ then echo "Please set $i" ; exit 1
+ fi
+done
+
+if [[ -z "$1" ]] || [[ -z "$2" ]]
+then echo "usage: <source-dir> <target-path>" ; exit 1
+fi
+
+FROM="$1"
+TO="$2"
+
+BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html"
+BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/"
+
+# install rclone
+if ! which rclone
+then curl https://rclone.org/install.sh | sudo bash
+fi
+
+RCLONE="rclone --config /dev/null \
+ --s3-provider AWS \
+ --s3-region $AWS_REGION \
+ --s3-acl public-read \
+ --s3-access-key-id $AWS_ACCESS_KEY_ID \
+ --s3-secret-access-key $AWS_SECRET_ACCESS_KEY"
+
+curl -s "$BROWSER" |\
+ sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\
+ $RCLONE rcat ":s3:$AWS_BUCKET/index.html"
+
+$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/"
diff --git a/tools/github/scan.sh b/tools/github/scan.sh
new file mode 100755
index 000000000..42b3fd573
--- /dev/null
+++ b/tools/github/scan.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# Build script for Travis-CI.
+SECONDS=0
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+HOMEDIR="$SCRIPTDIR/../../../"
+UTILDIR="$HOMEDIR/openwhisk-utilities/"
+
+cd $ROOTDIR
+./tools/github/flake8.sh # Check Python files for style and stop the build on syntax errors
+
+# clone the openwhisk utilities repo.
+cd $HOMEDIR
+git clone https://github.com/apache/openwhisk-utilities.git
+
+# run the scancode util. against project source code starting at its root
+cd $UTILDIR
+scancode/scanCode.py --config scancode/ASF-Release.cfg $ROOTDIR
+
+# run scalafmt checks
+cd $ROOTDIR
+TERM=dumb ./gradlew checkScalafmtAll
+
+# lint tests to all be actually runnable
+MISSING_TESTS=$(grep -rL "RunWith" --include="*Tests.scala" tests || true)
+if [ -n "$MISSING_TESTS" ]
+then
+ echo "The following tests are missing the 'RunWith' annotation"
+ echo $MISSING_TESTS
+ exit 1
+fi
+
+echo "Time taken for ${0##*/} is $SECONDS secs"
diff --git a/tools/github/setup.sh b/tools/github/setup.sh
new file mode 100755
index 000000000..e72b04605
--- /dev/null
+++ b/tools/github/setup.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#if [[ $TEST_SUITE =~ Dummy ]]
+#then echo skipping setup ; exit 0
+#fi
+
+# retries a command for five times and exits with the non-zero exit if even after
+# the retries the command did not succeed.
+function retry() {
+ local exitcode=0
+ for i in {1..5};
+ do
+ exitcode=0
+ "$@" && break || exitcode=$? && echo "$i. attempt failed. Will retry $((5-i)) more times!" && sleep 1;
+ done
+ if [ $exitcode -ne 0 ]; then
+ exit $exitcode
+ fi
+}
+
+# setup docker to listen in port 4243
+sudo systemctl stop docker
+sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service
+sudo systemctl daemon-reload
+sudo systemctl start docker
+
+# installing right version of jdk
+JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz
+curl -sL $JDK | sudo tar xzvf - -C /usr/local
+JAVA="$(which java)"
+sudo mv "$JAVA" "$JAVA"."$(date +%s)"
+sudo ln -sf /usr/local/jdk*/bin/java $JAVA
+java -version
+
+# Python
+python -m pip install --user couchdb
+
+# Ansible (warning you need jinja < 3.1 with this version)
+python -m pip install --user 'jinja2<3.1' ansible==2.8.18
+
+# Azure CosmosDB
+python -m pip install --user pydocumentdb
+
+# Support the revises log upload script
+python -m pip install --user humanize requests
+
+# Scan code before compiling the code
+tools/github/scan.sh
+
+# Preload alpine 3.5 to avoid issues with depending images
+retry docker pull alpine:3.5
+
+# exit if dummy test suite skipping the long compilation when debugging
+if [[ $TEST_SUITE =~ Dummy ]]
+then echo skiping setup ; exit 0
+fi
+
+# Basic check that all code compiles and dependencies are downloaded correctly.
+# Compiling the tests will compile all components as well.
+#
+# Downloads the gradle wrapper, dependencies and tries to compile the code.
+# Retried 5 times in case there are network hiccups.
+TERM=dumb retry ./gradlew :tests:compileTestScala
diff --git a/common/scala/Dockerfile b/tools/github/waitIfDebug.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/waitIfDebug.sh
index 6db867ff2..30f791b29
--- a/common/scala/Dockerfile
+++ b/tools/github/waitIfDebug.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +15,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+if ! test -e .ngrok.log
+then exit 0
+fi
+echo "You have an hour to debug this build."
+echo "Do touch /tmp/continue to continue."
+echo "Do touch /tmp/abort to abort."
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+EXIT=0
+for i in $(seq 1 60)
+do
+ if test -e /tmp/continue ; then EXIT=0 ; break ; fi
+ if test -e /tmp/abort ; then EXIT=1 ; break ; fi
+ echo "$i/60 still waiting..."
+ sleep 60
+done
-RUN mkdir /logs
-
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
-
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+killall ngrok
+rm -f .ngrok.log /tmp/continue /tmp/abort
+exit $EXIT
diff --git a/common/scala/Dockerfile b/tools/github/writeOnSlack.sh
old mode 100644
new mode 100755
similarity index 57%
copy from common/scala/Dockerfile
copy to tools/github/writeOnSlack.sh
index 6db867ff2..da8f2a8f2
--- a/common/scala/Dockerfile
+++ b/tools/github/writeOnSlack.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,20 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+if test -z "$SLACK_WEBHOOK"
+then echo "Please create an incoming webhook for slack and set SLACK_WEBHOOK"
+ exit 0
+fi
-# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
-RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
-RUN apk add --update sed curl bash && apk update && apk upgrade
+echo -n '{"text":' >/tmp/msg$$
+echo -n "$@" | jq -Rsa . >>/tmp/msg$$
+echo -n '}' >>/tmp/msg$$
-RUN mkdir /logs
-
-COPY transformEnvironment.sh /
-RUN chmod +x transformEnvironment.sh
-
-COPY copyJMXFiles.sh /
-RUN chmod +x copyJMXFiles.sh
+curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK"