You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/09/24 19:27:24 UTC
[accumulo] branch main updated: Create on-demand build for all ITs
as a matrix
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6986a42 Create on-demand build for all ITs as a matrix
6986a42 is described below
commit 6986a427fd78e82909e89dd079970e27c5922094
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Sep 24 15:18:09 2020 -0400
Create on-demand build for all ITs as a matrix
---
.../workflows/{maven.yaml => maven-full-its.yaml} | 42 ++++++++++++++--------
.github/workflows/maven-on-demand.yaml | 4 ++-
.github/workflows/maven.yaml | 3 +-
contrib/ci/it-matrix.sh | 38 ++++++++++++++++++++
4 files changed, 70 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven-full-its.yaml
similarity index 71%
copy from .github/workflows/maven.yaml
copy to .github/workflows/maven-full-its.yaml
index ee5f19c..658f717 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven-full-its.yaml
@@ -18,15 +18,20 @@
#
# This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#manual-events
+# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#fromjson
-name: QA
+name: Full ITs
on:
- push:
- branches: [ '*' ]
- pull_request:
- branches: [ '*' ]
+ workflow_dispatch:
+ inputs:
+ numITsPerTask:
+ description: Number of ITs per task
+ required: true
+ default: 15
jobs:
# fast build to populate the local maven repository cache
@@ -51,15 +56,22 @@ jobs:
run: mvn -B -V -e -ntp "-Dstyle.color=always" clean package dependency:resolve -PskipQA
env:
MAVEN_OPTS: -Djansi.force=true
- # more complete builds with tests
- mvn:
+ creatematrix:
needs: fastbuild
+ timeout-minutes: 5
+ outputs:
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - id: set-matrix
+ name: Create the IT build matrix
+ run: contrib/ci/it-matrix.sh ${{ github.event.inputs.numITsPerTask }}
+ # targeted builds that tests groups of ITs
+ mvn:
+ needs: creatematrix
strategy:
- matrix:
- profile:
- - {name: 'unit-tests', args: 'verify -PskipQA -DskipTests=false -DforkCount=1C'}
- - {name: 'qa-checks', args: 'verify javadoc:jar -Psec-bugs -DskipTests=true -Dspotbugs.timeout=3600000'}
- - {name: 'hadoop-compat', args: 'package -DskipTests -Dhadoop.version=3.0.3'}
+ matrix: ${{ fromJson(needs.creatematrix.outputs.matrix) }}
fail-fast: false
timeout-minutes: 60
runs-on: ubuntu-latest
@@ -77,8 +89,8 @@ jobs:
!~/.m2/repository/org/apache/accumulo
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
- - name: Build with Maven (${{ matrix.profile.name }})
- run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ matrix.profile.args }}
+ - name: Build with Maven (${{ matrix.profile.its }})
+ run: mvn -B -V -e -ntp "-Dstyle.color=always" verify -PskipQA -DskipTests=false -DskipITs=false -Dtest=nomatchingtest -Dit.test="${{ matrix.profile.its }}"
env:
MAVEN_OPTS: -Djansi.force=true
- name: Upload unit test results
diff --git a/.github/workflows/maven-on-demand.yaml b/.github/workflows/maven-on-demand.yaml
index 34af0df..ef50d43 100644
--- a/.github/workflows/maven-on-demand.yaml
+++ b/.github/workflows/maven-on-demand.yaml
@@ -18,7 +18,9 @@
#
# This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#manual-events
name: Manual Build
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
index ee5f19c..b52420f 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@ -18,7 +18,8 @@
#
# This workflow will build a Java project with Maven
-# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# See also:
+# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
name: QA
diff --git a/contrib/ci/it-matrix.sh b/contrib/ci/it-matrix.sh
new file mode 100755
index 0000000..89a887d
--- /dev/null
+++ b/contrib/ci/it-matrix.sh
@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+testsPerJob=15
+if [[ -n $1 && $1 =~ ^[0-9]*$ ]]; then
+ testsPerJob=$1
+fi
+echo "Creating matrix (tests per job: $testsPerJob)..."
+
+gitRootDir=$(git rev-parse --show-toplevel)
+# this only works because our test paths don't have spaces; we should keep it that way
+count=0
+echo -n '::set-output name=matrix::{"profile":['
+for x in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; | sort -u | xargs -n "$testsPerJob" | tr ' ' ','); do
+ [[ $count -gt 0 ]] && echo -n ','
+ echo -n "{\"name\":\"task_$count\",\"its\":\"$x\"}"
+ ((count=count+1))
+done
+echo ']}'
+echo "Finished creating matrix ($count tasks)"
+