You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ja...@apache.org on 2023/10/02 13:45:31 UTC
[camel-quarkus] branch main updated: Speed up examples CI build step
This is an automated email from the ASF dual-hosted git repository.
jamesnetherton pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
The following commit(s) were added to refs/heads/main by this push:
new e2e32c9285 Speed up examples CI build step
e2e32c9285 is described below
commit e2e32c92853606cf2fefc2dd29ffa55b3342193a
Author: James Netherton <ja...@gmail.com>
AuthorDate: Fri Sep 29 14:02:13 2023 +0100
Speed up examples CI build step
Fixes #5136
---
.github/workflows/camel-master-cron.yaml | 51 +++++++++++++++++++++++-
.github/workflows/ci-build.yaml | 49 ++++++++++++++++++++++-
.github/workflows/quarkus-master-cron.yaml | 51 +++++++++++++++++++++++-
tooling/scripts/generate-examples-matrix.groovy | 53 +++++++++++++++++++++++++
4 files changed, 198 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/camel-master-cron.yaml b/.github/workflows/camel-master-cron.yaml
index 654fb422f5..9e54c89deb 100644
--- a/.github/workflows/camel-master-cron.yaml
+++ b/.github/workflows/camel-master-cron.yaml
@@ -35,6 +35,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-native-matrix.outputs.matrix }}
+ examples-matrix: ${{ steps.set-examples-matrix.outputs.examples-matrix }}
env:
MAVEN_OPTS: -Xmx4400m
steps:
@@ -96,6 +97,19 @@ jobs:
run: |
CATEGORIES=$(yq -M -N -I 0 -o=json e 'keys' tooling/scripts/test-categories.yaml | tr '"' "'")
echo "matrix={'category': ${CATEGORIES}}" >> $GITHUB_OUTPUT
+ - name: Setup Examples Matrix
+ id: set-examples-matrix
+ run: |
+ EXAMPLES_BRANCH="camel-quarkus-main"
+ if [[ ${GITHUB_REF_NAME} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_REF_NAME}
+ elif [[ ${GITHUB_BASE_REF} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_BASE_REF}
+ fi
+
+ sudo apt install groovy -y --no-install-recommends
+ EXAMPLES_MATRIX=$(groovy -DEXAMPLES_BRANCH=${EXAMPLES_BRANCH} tooling/scripts/generate-examples-matrix.groovy)
+ echo "examples-matrix=${EXAMPLES_MATRIX}" >> $GITHUB_OUTPUT
- name: Report Build Failure
if: failure() || cancelled()
run: |
@@ -373,8 +387,12 @@ jobs:
./mvnw ${CQ_MAVEN_ARGS} verify -N -Pbuild-notification -Dstatus=${{ job.status }} -DissueId=${{ env.ISSUE_ID }} -Dtoken=${{ secrets.GITHUB_TOKEN }} -DbuildId=$(cat ~/build-data/build-id.txt) -Drepo=${GITHUB_REPOSITORY} -Dbranch=camel-main -Dbranch-commit=$(cat ~/build-data/main-sha.txt)
examples-tests:
+ name: Examples Tests - ${{matrix.name}}
needs: initial-mvn-install
runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.initial-mvn-install.outputs.examples-matrix) }}
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
@@ -406,18 +424,47 @@ jobs:
- name: set CQ_VERSION
run: echo "CQ_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout -N)" >> $GITHUB_ENV
- name: clone and verify examples
+ env:
+ EXAMPLE_MODULES: ${{matrix.examples}}
+ shell: '/usr/bin/bash {0}'
run: |
EXAMPLES_BRANCH="camel-quarkus-main"
if [[ ${GITHUB_REF_NAME} =~ [0-9]+.[0-9]+.x ]]; then
EXAMPLES_BRANCH=${GITHUB_REF_NAME}
+ elif [[ ${GITHUB_BASE_REF} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_BASE_REF}
fi
git clone --depth 1 --branch ${EXAMPLES_BRANCH} https://github.com/apache/camel-quarkus-examples.git \
&& cd camel-quarkus-examples \
&& echo "Current Examples commit:" $(git rev-parse HEAD) \
- && ./mvnw ${CQ_MAVEN_ARGS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION} \
- && ./mvnw-for-each.sh ${CQ_MAVEN_ARGS} -Pnative,docker clean verify
+ && ./mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION}
+
+ BUILD_FAILURES=()
+
+ for MODULE in ${EXAMPLE_MODULES//,/ }; do
+ cd ${MODULE}
+
+ ../mvnw ${MAVEN_ARGS} clean verify \
+ -Dformatter.skip -Dimpsort.skip \
+ -Pnative,docker,ci
+
+ if [[ $? -ne 0 ]]; then
+ BUILD_FAILURES[${#BUILD_FAILURES[@]}]=${MODULE}
+ fi
+
+ cd -
+ done
+
+ if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then
+ echo -e "\nBuild errors were encountred in the following projects:\n"
+ for FAILURE in ${BUILD_FAILURES[@]}; do
+ echo "* ${FAILURE}"
+ done
+ echo -e "\nCheck build logs for further information."
+ exit 1
+ fi
- name: Report Build Failure
if: failure() || cancelled()
run: |
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
index 318c399466..3e206dd2d5 100644
--- a/.github/workflows/ci-build.yaml
+++ b/.github/workflows/ci-build.yaml
@@ -123,6 +123,7 @@ jobs:
needs: pre-build-checks
outputs:
matrix: ${{ steps.set-native-matrix.outputs.matrix }}
+ examples-matrix: ${{ steps.set-examples-matrix.outputs.examples-matrix }}
env:
MAVEN_OPTS: -Xmx4400m
steps:
@@ -185,6 +186,19 @@ jobs:
run: |
CATEGORIES=$(yq -M -N -I 0 -o=json e 'keys' tooling/scripts/test-categories.yaml | tr '"' "'")
echo "matrix={'category': ${CATEGORIES}}" >> $GITHUB_OUTPUT
+ - name: Setup Examples Matrix
+ id: set-examples-matrix
+ run: |
+ EXAMPLES_BRANCH="camel-quarkus-main"
+ if [[ ${GITHUB_REF_NAME} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_REF_NAME}
+ elif [[ ${GITHUB_BASE_REF} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_BASE_REF}
+ fi
+
+ sudo apt install groovy -y --no-install-recommends
+ EXAMPLES_MATRIX=$(groovy -DEXAMPLES_BRANCH=${EXAMPLES_BRANCH} tooling/scripts/generate-examples-matrix.groovy)
+ echo "examples-matrix=${EXAMPLES_MATRIX}" >> $GITHUB_OUTPUT
native-tests:
name: Native Tests - ${{matrix.category}}
@@ -432,9 +446,13 @@ jobs:
clean verify
examples-tests:
+ name: Examples Tests - ${{matrix.name}}
needs: initial-mvn-install
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'JVM')
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.initial-mvn-install.outputs.examples-matrix) }}
steps:
- name: Setup oss-snapshots profile
if: github.ref == 'refs/heads/quarkus-main' || github.base_ref == 'quarkus-main'
@@ -461,6 +479,9 @@ jobs:
- name: set CQ_VERSION
run: echo "CQ_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout -N)" >> $GITHUB_ENV
- name: clone and verify examples
+ env:
+ EXAMPLE_MODULES: ${{matrix.examples}}
+ shell: '/usr/bin/bash {0}'
run: |
EXAMPLES_BRANCH="camel-quarkus-main"
@@ -473,5 +494,29 @@ jobs:
git clone --depth 1 --branch ${EXAMPLES_BRANCH} https://github.com/apache/camel-quarkus-examples.git \
&& cd camel-quarkus-examples \
&& echo "Current Examples commit:" $(git rev-parse HEAD) \
- && mvn ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION} \
- && ./mvnw-for-each.sh ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} -Pnative,docker clean verify
+ && ./mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION}
+
+ BUILD_FAILURES=()
+
+ for MODULE in ${EXAMPLE_MODULES//,/ }; do
+ cd ${MODULE}
+
+ ../mvnw ${MAVEN_ARGS} clean verify \
+ -Dformatter.skip -Dimpsort.skip \
+ -Pnative,docker,ci
+
+ if [[ $? -ne 0 ]]; then
+ BUILD_FAILURES[${#BUILD_FAILURES[@]}]=${MODULE}
+ fi
+
+ cd -
+ done
+
+ if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then
+ echo -e "\nBuild errors were encountred in the following projects:\n"
+ for FAILURE in ${BUILD_FAILURES[@]}; do
+ echo "* ${FAILURE}"
+ done
+ echo -e "\nCheck build logs for further information."
+ exit 1
+ fi
diff --git a/.github/workflows/quarkus-master-cron.yaml b/.github/workflows/quarkus-master-cron.yaml
index cce88db8b6..55a9137aa7 100644
--- a/.github/workflows/quarkus-master-cron.yaml
+++ b/.github/workflows/quarkus-master-cron.yaml
@@ -35,6 +35,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-native-matrix.outputs.matrix }}
+ examples-matrix: ${{ steps.set-examples-matrix.outputs.examples-matrix }}
env:
MAVEN_OPTS: -Xmx4400m
steps:
@@ -96,6 +97,19 @@ jobs:
run: |
CATEGORIES=$(yq -M -N -I 0 -o=json e 'keys' tooling/scripts/test-categories.yaml | tr '"' "'")
echo "matrix={'category': ${CATEGORIES}}" >> $GITHUB_OUTPUT
+ - name: Setup Examples Matrix
+ id: set-examples-matrix
+ run: |
+ EXAMPLES_BRANCH="camel-quarkus-main"
+ if [[ ${GITHUB_REF_NAME} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_REF_NAME}
+ elif [[ ${GITHUB_BASE_REF} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_BASE_REF}
+ fi
+
+ sudo apt install groovy -y --no-install-recommends
+ EXAMPLES_MATRIX=$(groovy -DEXAMPLES_BRANCH=${EXAMPLES_BRANCH} tooling/scripts/generate-examples-matrix.groovy)
+ echo "examples-matrix=${EXAMPLES_MATRIX}" >> $GITHUB_OUTPUT
- name: Report Build Failure
if: failure() || cancelled()
run: |
@@ -373,8 +387,12 @@ jobs:
./mvnw ${CQ_MAVEN_ARGS} verify -N -Pbuild-notification -Dstatus=${{ job.status }} -DissueId=${{ env.ISSUE_ID }} -Dtoken=${{ secrets.GITHUB_TOKEN }} -DbuildId=$(cat ~/build-data/build-id.txt) -Drepo=${GITHUB_REPOSITORY} -Dbranch=quarkus-main -Dbranch-commit=$(cat ~/build-data/main-sha.txt)
examples-tests:
+ name: Examples Tests - ${{matrix.name}}
needs: initial-mvn-install
runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.initial-mvn-install.outputs.examples-matrix) }}
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
@@ -406,18 +424,47 @@ jobs:
- name: set CQ_VERSION
run: echo "CQ_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout -N)" >> $GITHUB_ENV
- name: clone and verify examples
+ env:
+ EXAMPLE_MODULES: ${{matrix.examples}}
+ shell: '/usr/bin/bash {0}'
run: |
EXAMPLES_BRANCH="camel-quarkus-main"
if [[ ${GITHUB_REF_NAME} =~ [0-9]+.[0-9]+.x ]]; then
EXAMPLES_BRANCH=${GITHUB_REF_NAME}
+ elif [[ ${GITHUB_BASE_REF} =~ [0-9]+.[0-9]+.x ]]; then
+ EXAMPLES_BRANCH=${GITHUB_BASE_REF}
fi
git clone --depth 1 --branch ${EXAMPLES_BRANCH} https://github.com/apache/camel-quarkus-examples.git \
&& cd camel-quarkus-examples \
&& echo "Current Examples commit:" $(git rev-parse HEAD) \
- && ./mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION} \
- && ./mvnw-for-each.sh ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} -Pnative,docker clean verify
+ && ./mvnw ${CQ_MAVEN_ARGS} ${BRANCH_OPTIONS} org.l2x6.cq:cq-maven-plugin:2.10.0:examples-set-platform -Dcq.camel-quarkus.version=${CQ_VERSION}
+
+ BUILD_FAILURES=()
+
+ for MODULE in ${EXAMPLE_MODULES//,/ }; do
+ cd ${MODULE}
+
+ ../mvnw ${MAVEN_ARGS} clean verify \
+ -Dformatter.skip -Dimpsort.skip \
+ -Pnative,docker,ci
+
+ if [[ $? -ne 0 ]]; then
+ BUILD_FAILURES[${#BUILD_FAILURES[@]}]=${MODULE}
+ fi
+
+ cd -
+ done
+
+ if [[ ${#BUILD_FAILURES[@]} -gt 0 ]]; then
+ echo -e "\nBuild errors were encountred in the following projects:\n"
+ for FAILURE in ${BUILD_FAILURES[@]}; do
+ echo "* ${FAILURE}"
+ done
+ echo -e "\nCheck build logs for further information."
+ exit 1
+ fi
- name: Report Build Failure
if: failure() || cancelled()
run: |
diff --git a/tooling/scripts/generate-examples-matrix.groovy b/tooling/scripts/generate-examples-matrix.groovy
new file mode 100644
index 0000000000..ff52c26116
--- /dev/null
+++ b/tooling/scripts/generate-examples-matrix.groovy
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import groovy.json.JsonOutput;
+import groovy.json.JsonSlurper
+
+final int MAX_GROUPS = 2
+final List<Map<String, String>> GROUPS = new ArrayList<>()
+final String EXAMPLES_BRANCH = System.getProperty('EXAMPLES_BRANCH')
+
+int groupId = 0
+JsonSlurper jsonSlurper = new JsonSlurper()
+
+try {
+ def url = new URL("https://raw.githubusercontent.com/apache/camel-quarkus-examples/${EXAMPLES_BRANCH}/docs/modules/ROOT/attachments/examples.json")
+ def examples = jsonSlurper.parse(url)
+
+ // Distribute example projects across a bounded set of test groups and output as JSON
+ examples.each { example ->
+ if (GROUPS[groupId] == null) {
+ GROUPS[groupId] = [:]
+ GROUPS[groupId].name = "group-${String.format("%02d", groupId + 1)}"
+ GROUPS[groupId].examples = ""
+ }
+
+ String separator = GROUPS[groupId].examples == "" ? "" : ","
+ String projectName = example.link.substring(example.link.lastIndexOf('/') + 1)
+
+ GROUPS[groupId].examples = "${GROUPS[groupId].examples}${separator}${projectName}"
+
+ groupId += 1;
+ if (groupId == MAX_GROUPS) {
+ groupId = 0
+ }
+ }
+} catch (Exception e) {
+ // Ignored
+}
+
+print JsonOutput.toJson(["include": GROUPS])