You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by nf...@apache.org on 2023/06/08 19:59:32 UTC
[camel] branch main updated: Add an incremental build (#10253)
This is an automated email from the ASF dual-hosted git repository.
nfilotto pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 4d73e6398b4 Add an incremental build (#10253)
4d73e6398b4 is described below
commit 4d73e6398b458ecf8ce7531cf55680e7e152f171
Author: Nicolas Filotto <es...@users.noreply.github.com>
AuthorDate: Thu Jun 8 21:59:25 2023 +0200
Add an incremental build (#10253)
## Motivation
The build launched for each PR is much too long, let's experiment with an incremental build. The idea is to build only what is needed.
## Modifications:
* Adds 2 new jobs that are the counterpart of `checkstyle` and `build` in incremental mode. As they are experimental, if they fail, it won't block the merge
* Builds everything like before if there are too many projects affected
* Launches the checkstyle command only on the affected projects
* Launches the fast build on the affected projects, their dependencies, and eventually the projects that depend on them
* Launches the test on the affected projects, and eventually the projects that depend on them
* Ignores changes in a folder whose name starts with a dot like `.github`
---
.github/actions/incremental-build/action.yaml | 46 ++++++++++
.../actions/incremental-build/incremental-build.sh | 97 ++++++++++++++++++++++
.github/actions/install-mvnd/action.yml | 10 ++-
.github/workflows/pr-build-main.yml | 47 +++++++++++
4 files changed, 197 insertions(+), 3 deletions(-)
diff --git a/.github/actions/incremental-build/action.yaml b/.github/actions/incremental-build/action.yaml
new file mode 100644
index 00000000000..04f83b535e6
--- /dev/null
+++ b/.github/actions/incremental-build/action.yaml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: "Incremental Build Runner"
+description: "Build only affected projects"
+inputs:
+ checkstyle:
+ description: 'Flag indicating whether only checkstyle should be launched'
+ required: true
+ log:
+ description: 'Name of the log file'
+ required: true
+ pr-id:
+ description: 'Id of the pull request'
+ required: true
+runs:
+ using: "composite"
+ steps:
+ - id: install-mvnd
+ uses: ./.github/actions/install-mvnd
+ with:
+ version: 1.0-m6
+ distribution: m39-linux-amd64
+ - name: maven build
+ shell: bash
+ run: ${{ github.action_path }}/incremental-build.sh ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd ${{ inputs.checkstyle }} ${{ inputs.log }} ${{ inputs.pr-id }}
+ - name: archive logs
+ uses: actions/upload-artifact@v3
+ if: always()
+ with:
+ name: ${{ inputs.log }}
+ path: ${{ inputs.log }}
diff --git a/.github/actions/incremental-build/incremental-build.sh b/.github/actions/incremental-build/incremental-build.sh
new file mode 100755
index 00000000000..47cc2d9fbcf
--- /dev/null
+++ b/.github/actions/incremental-build/incremental-build.sh
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Modify maven options here if needed
+MVN_DEFAULT_OPTS="-Dmvnd.threads=2 -V -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 --no-transfer-progress -e -Dmaven.artifact.threads=25 -Daether.dependencyCollector.impl=bf"
+MVN_OPTS=${MVN_OPTS:-$MVN_DEFAULT_OPTS}
+
+maxNumberOfBuildableProjects=100
+maxNumberOfTestableProjects=50
+
+function findProjectRoot () {
+ local path=${1}
+ while [[ "$path" != "." && ! -e "$path/pom.xml" ]]; do
+ path=$(dirname $path)
+ done
+ echo "$path"
+}
+
+function main() {
+ local mavenBinary=${1}
+ local checkstyle=${2}
+ local log=${3}
+ local prId=${4}
+
+ echo "Searching for affected projects"
+ local projects=$(curl -s "https://patch-diff.githubusercontent.com/raw/apache/camel/pull/${prId}.diff" | sed -n -e '/^diff --git a/p' | awk '{print $3}' | cut -b 3- | sed 's|\(.*\)/.*|\1|' | uniq | sort)
+ local pl=""
+ local lastProjectRoot=""
+ local buildAll=false
+ local totalAffected=0
+ for project in ${projects}
+ do
+ if [[ ${project} != .* ]] ; then
+ projectRoot=$(findProjectRoot ${project})
+ if [[ ${projectRoot} = "." ]] ; then
+ echo "There root project is affected, so a complete build is triggered"
+ buildAll=true
+ elif [[ ${projectRoot} != "${lastProjectRoot}" ]] ; then
+ (( totalAffected ++ ))
+ pl="$pl,${projectRoot}"
+ fi
+ fi
+ done
+ if [[ ${totalAffected} = 0 ]] ; then
+ echo "There is nothing to build"
+ exit 0
+ elif [[ ${totalAffected} -gt ${maxNumberOfBuildableProjects} ]] ; then
+ echo "There are too many affected projects, so a complete build is triggered"
+ buildAll=true
+ fi
+ pl="${pl:1}"
+
+ if [[ ${checkstyle} = "true" ]] ; then
+ if [[ ${buildAll} = "true" ]] ; then
+ echo "Launching checkstyle command against all projects"
+ $mavenBinary -l $log $MVN_OPTS -Dcheckstyle.failOnViolation=true checkstyle:checkstyle
+ else
+ echo "Launching checkstyle command against the projects ${pl}"
+ $mavenBinary -l $log $MVN_OPTS -Dcheckstyle.failOnViolation=true checkstyle:checkstyle -pl "$pl"
+ fi
+ else
+ if [[ ${buildAll} = "true" ]] ; then
+ echo "Launching fast build command against all projects"
+ $mavenBinary -l $log $MVN_OPTS -Pfastinstall install
+ echo "Cannot launch the tests of all projects, so no test will be launched"
+ else
+ local totalTestableProjects=$(mvn -q -amd exec:exec -Dexec.executable="pwd" -pl "$pl" | wc -l)
+ if [[ ${totalTestableProjects} -gt ${maxNumberOfTestableProjects} ]] ; then
+ echo "Launching fast build command against the projects ${pl}, their dependencies and the projects that depend on them"
+ $mavenBinary $MVN_OPTS -Pfastinstall install -pl "$pl" -amd -am >> $log
+ echo "There are too many projects to test so only the affected projects are tested"
+ $mavenBinary $MVN_OPTS install -pl "$pl" >> $log
+ else
+ echo "Launching fast build command against the projects ${pl} and their dependencies"
+ $mavenBinary $MVN_OPTS -Pfastinstall install -pl "$pl" -am >> $log
+ echo "Testing the affected projects and the projects that depend on them"
+ $mavenBinary $MVN_OPTS install -pl "$pl" -amd >> $log
+ fi
+ fi
+ fi
+}
+
+main "$@"
diff --git a/.github/actions/install-mvnd/action.yml b/.github/actions/install-mvnd/action.yml
index 5414102c1b9..3035c3c3f50 100644
--- a/.github/actions/install-mvnd/action.yml
+++ b/.github/actions/install-mvnd/action.yml
@@ -22,6 +22,10 @@ inputs:
description: 'The version of the maven daemon to install'
required: true
default: '0.8.2'
+ distribution:
+ description: 'The maven distribution to use'
+ required: true
+ default: 'linux-amd64'
outputs:
mvnd-dir:
description: "The directory where the command mvnd is located"
@@ -29,9 +33,9 @@ outputs:
runs:
using: "composite"
steps:
- - run: curl -fsSL -o mvnd.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/maven-mvnd-${{ inputs.version }}-linux-amd64.zip
+ - run: curl -fsSL -o mvnd.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/maven-mvnd-${{ inputs.version }}-${{ inputs.distribution }}.zip
shell: bash
- - run: curl -fsSL -o mvnd.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/maven-mvnd-${{ inputs.version }}-linux-amd64.zip.sha256
+ - run: curl -fsSL -o mvnd.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/maven-mvnd-${{ inputs.version }}-${{ inputs.distribution }}.zip.sha256
shell: bash
- id: integrity-check
run: echo "$(cat mvnd.zip.sha256) mvnd.zip" | sha256sum --check
@@ -39,5 +43,5 @@ runs:
- run: unzip mvnd.zip -d /tmp/
shell: bash
- id: mvnd-location
- run: echo "mvnd-dir=/tmp/maven-mvnd-${{ inputs.version }}-linux-amd64/bin" >> $GITHUB_OUTPUT
+ run: echo "mvnd-dir=/tmp/maven-mvnd-${{ inputs.version }}-${{ inputs.distribution }}/bin" >> $GITHUB_OUTPUT
shell: bash
diff --git a/.github/workflows/pr-build-main.yml b/.github/workflows/pr-build-main.yml
index 55dc4b599f1..f33a3057bfa 100644
--- a/.github/workflows/pr-build-main.yml
+++ b/.github/workflows/pr-build-main.yml
@@ -100,3 +100,50 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
start-commit: ${{ github.event.pull_request.base.sha }}
end-commit: ${{ github.event.after }}
+ incremental-checkstyle:
+ if: github.repository == 'apache/camel'
+ # Mark as optional since it is experimental
+ continue-on-error: true
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: Set up JDK
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 17
+ cache: 'maven'
+ - name: mvn checkstyle
+ uses: ./.github/actions/incremental-build
+ with:
+ checkstyle: true
+ log: incremental-checkstyle.log
+ pr-id: ${{ github.event.number }}
+ incremental-build:
+ if: github.repository == 'apache/camel'
+ # Mark as optional since it is experimental
+ continue-on-error: true
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ java: [ '17' ]
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - id: install-packages
+ uses: ./.github/actions/install-packages
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.java }}
+ cache: 'maven'
+ - name: maven build
+ uses: ./.github/actions/incremental-build
+ with:
+ checkstyle: false
+ log: incremental-build.log
+ pr-id: ${{ github.event.number }}