You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2020/03/09 12:31:50 UTC
[camel-quarkus] branch master updated: Optimize PR build GitHub
action
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
The following commit(s) were added to refs/heads/master by this push:
new 92f1e07 Optimize PR build GitHub action
new e544f55 Merge pull request #861 from jamesnetherton/improve-actions
92f1e07 is described below
commit 92f1e07059b08439d4e1b6e057341d6fb1df984f
Author: James Netherton <ja...@gmail.com>
AuthorDate: Mon Mar 9 10:59:07 2020 +0000
Optimize PR build GitHub action
Fixes #836
---
.github/workflows/pr-build.yaml | 442 ++++++-----------------
.github/workflows/pr-validate.yml | 5 +
pom.xml | 7 +
tooling/scripts/validate-github-workflows.groovy | 27 +-
4 files changed, 127 insertions(+), 354 deletions(-)
diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml
index 2a51e51..dd0b376 100644
--- a/.github/workflows/pr-build.yaml
+++ b/.github/workflows/pr-build.yaml
@@ -23,6 +23,11 @@ on:
- master
- camel-master
- quarkus-master
+ paths-ignore:
+ - '**.adoc'
+ - 'KEYS'
+ - 'LICENSE.txt'
+ - 'NOTICE.txt'
env:
LANG: en_US
@@ -81,347 +86,101 @@ jobs:
./mvnw -V -B ${BRANCH_OPTIONS} \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
clean verify
- core:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-core \
- -pl :camel-quarkus-integration-test-core-impl \
- -pl :camel-quarkus-integration-test-core-main \
- -pl :camel-quarkus-integration-test-core-main-collector \
- -pl :camel-quarkus-integration-test-core-main-xml-jaxb \
- -pl :camel-quarkus-integration-test-core-main-xml-io \
- foundation:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-bean \
- -pl :camel-quarkus-integration-test-seda \
- -pl :camel-quarkus-integration-test-file \
- -pl :camel-quarkus-integration-test-scheduler \
- -pl :camel-quarkus-integration-test-jsonpath \
- -pl :camel-quarkus-integration-test-exec \
- -pl :camel-quarkus-integration-test-controlbus \
- -pl :camel-quarkus-integration-test-hystrix
- platform:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-platform-http \
- -pl :camel-quarkus-integration-test-platform-http-engine \
- -pl :camel-quarkus-integration-test-microprofile \
- -pl :camel-quarkus-integration-test-opentracing \
- -pl :camel-quarkus-integration-test-reactive-streams
- messaging:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-activemq \
- -pl :camel-quarkus-integration-test-kafka \
- -pl :camel-quarkus-integration-test-messaging
- dataformats:
- runs-on: ubuntu-latest
+ native-tests:
+ name: Native Tests - ${{matrix.category}}
needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-base64 \
- -pl :camel-quarkus-integration-test-bindy \
- -pl :camel-quarkus-integration-test-csv \
- -pl :camel-quarkus-integration-test-dataformat \
- -pl :camel-quarkus-integration-test-jaxb \
- -pl :camel-quarkus-integration-test-xstream
- networking:
runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-http \
- -pl :camel-quarkus-integration-test-netty \
- -pl :camel-quarkus-integration-test-servlet \
- -pl :camel-quarkus-integration-test-websocket-jsr356
- sql:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-jdbc \
- -pl :camel-quarkus-integration-test-mongodb \
- -pl :camel-quarkus-integration-test-sql
- cloud:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-aws \
- -pl :camel-quarkus-integration-test-azure \
- -pl :camel-quarkus-integration-test-consul
- misc:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-bean-validator \
- -pl :camel-quarkus-integration-test-braintree \
- -pl :camel-quarkus-integration-test-dozer \
- -pl :camel-quarkus-integration-test-ftp \
- -pl :camel-quarkus-integration-test-mail \
- -pl :camel-quarkus-integration-test-pdf \
- -pl :camel-quarkus-integration-test-stream \
- -pl :camel-quarkus-integration-test-telegram \
- -pl :camel-quarkus-integration-test-tarfile \
- -pl :camel-quarkus-integration-test-validator \
- -pl :camel-quarkus-integration-test-compression
- saas:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Set BRANCH_OPTIONS
- if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
- run: |
- echo '::set-env name=BRANCH_OPTIONS::-Psnapshots'
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set Up Java
- uses: actions/setup-java@v1
- with:
- java-version: 1.8
- - name: Restore Cache
- uses: actions/cache@v1
- with:
- path: ~/.m2/repository
- key: maven-${{ github.sha }}
- restore-keys: |
- maven-${{ github.sha }}
- - name: Integration Tests
- run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
- -Dnative \
- -Ddocker \
- -DskipTests \
- -pl :camel-quarkus-integration-test-box \
- -pl :camel-quarkus-integration-test-jira \
- -pl :camel-quarkus-integration-test-salesforce \
- -pl :camel-quarkus-integration-test-slack \
- -pl :camel-quarkus-integration-test-twitter
- # Do slow (> 10 minute) native builds in parallel
- memoryhogs:
- runs-on: ubuntu-latest
- needs: build
strategy:
+ fail-fast: false
matrix:
- extension: [ 'dataformats-json', 'infinispan', 'olingo4', 'xml' ]
+ category: [Core, Foundation, Platform, Messaging, Dataformats, Networking, SQL, Cloud, Misc, SAAS, Memoryhog Dataformat JSON, Memoryhog Infinispan, Memoryhog Olingo4, Memoryhog XML]
+ include:
+ - category: Cloud
+ test-modules: >
+ aws
+ azure
+ consul
+ - category: Core
+ test-modules: >
+ core
+ core-impl
+ core-main
+ core-main-collector
+ core-main-xml-io
+ core-main-xml-jaxb
+ - category: Dataformats
+ test-modules: >
+ base64
+ bindy
+ csv
+ dataformat
+ jaxb
+ xstream
+ - category: Foundation
+ test-modules: >
+ bean
+ controlbus
+ exec
+ file
+ hystrix
+ jsonpath
+ scheduler
+ seda
+ - category: Memoryhog Dataformat JSON
+ test-modules: >
+ dataformats-json
+ - category: Memoryhog Infinispan
+ test-modules: >
+ infinispan
+ - category: Memoryhog Olingo4
+ test-modules: >
+ olingo4
+ - category: Memoryhog XML
+ test-modules: >
+ xml
+ - category: Messaging
+ test-modules: >
+ activemq
+ kafka
+ messaging
+ - category: Misc
+ test-modules: >
+ bean-validator
+ braintree
+ compression
+ dozer
+ ftp
+ mail
+ pdf
+ stream
+ tarfile
+ telegram
+ validator
+ - category: Networking
+ test-modules: >
+ http
+ netty
+ servlet
+ websocket-jsr356
+ - category: Platform
+ test-modules: >
+ microprofile
+ opentracing
+ platform-http
+ platform-http-engine
+ reactive-streams
+ - category: SAAS
+ test-modules: >
+ box
+ jira
+ salesforce
+ slack
+ twitter
+ - category: SQL
+ test-modules: >
+ jdbc
+ mongodb
+ sql
steps:
- name: Set BRANCH_OPTIONS
if: github.base_ref == 'camel-master' || github.base_ref == 'quarkus-master'
@@ -441,13 +200,18 @@ jobs:
restore-keys: |
maven-${{ github.sha }}
- name: Integration Tests
+ env:
+ TEST_MODULES: ${{matrix.test-modules}}
run: |
- ./mvnw -V -B ${BRANCH_OPTIONS} \
+ for i in $TEST_MODULES
+ do modules+=("integration-tests/$i"); done
+ IFS=,
+ eval ./mvnw -V -B ${BRANCH_OPTIONS} \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean verify \
-Dnative \
-Ddocker \
-DskipTests \
- -pl :camel-quarkus-integration-test-${{ matrix.extension }}
+ -pl "${modules[*]}"
# memoryhogs:
# runs-on: ubuntu-latest
diff --git a/.github/workflows/pr-validate.yml b/.github/workflows/pr-validate.yml
index 87e5dab..75ebc11 100644
--- a/.github/workflows/pr-validate.yml
+++ b/.github/workflows/pr-validate.yml
@@ -23,6 +23,11 @@ on:
- master
- camel-master
- quarkus-master
+ paths-ignore:
+ - '**.adoc'
+ - 'KEYS'
+ - 'LICENSE.txt'
+ - 'NOTICE.txt'
env:
LANG: en_US
diff --git a/pom.xml b/pom.xml
index 8eb691c..b51841a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,6 +349,13 @@
</configuration>
</execution>
</executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+ </dependencies>
</plugin>
</plugins>
</build>
diff --git a/tooling/scripts/validate-github-workflows.groovy b/tooling/scripts/validate-github-workflows.groovy
index db0da5e..83b3bd5 100644
--- a/tooling/scripts/validate-github-workflows.groovy
+++ b/tooling/scripts/validate-github-workflows.groovy
@@ -19,33 +19,30 @@
* Makes sure that each itest is executed by the CI
*/
import java.nio.file.Path
-import java.nio.file.Paths
import java.nio.file.Files
-import java.util.regex.Matcher
-import java.util.regex.Pattern
-import java.util.stream.Collectors
+import org.yaml.snakeyaml.Yaml
final Path treeRootDir = project.getBasedir().toPath()
final String jobDefRelPath = '.github/workflows/pr-build.yaml'
final Path jobDefPath = treeRootDir.resolve(jobDefRelPath)
-final String jobDefSource = new String(Files.readAllBytes(jobDefPath), 'UTF-8')
-
final Set<String> executedBaseNames = [] as Set
-final Matcher plMatcher = Pattern.compile('-pl :camel-quarkus-integration-test-([^\n ]+)').matcher(jobDefSource)
-while (plMatcher.find()) {
- executedBaseNames.add(plMatcher.group(1))
-}
-final Matcher extensionMatcher = Pattern.compile('extension: *\\[ *\'([^\\]]+)\' *\\]').matcher(jobDefSource)
-while (extensionMatcher.find()) {
- extensionMatcher.group(1).split('\' *, *\'').each { executedBaseNames.add(it) }
-}
+
+// Add any ignored itest modules here. Or prefix the module name with '#' to disable it
+final List<String> excludedModules = ['fhir', 'support'] as List
+
+final Yaml parser = new Yaml()
+def prConfig = parser.load((jobDefPath.toFile()).text)
+
+modules = prConfig['jobs']['native-tests']['strategy']['matrix']['include']['test-modules']
+modules.each { executedBaseNames.addAll(it.trim().split(' ')) }
final Set<String> missingBaseNames = [] as TreeSet
final Set<String> itestBaseNames = Files.list(treeRootDir.resolve('integration-tests'))
.filter{ path -> Files.exists(path.resolve('pom.xml')) }
.map{ path -> path.getFileName().toString() }
- .filter{ dirName -> !dirName.equals('support') }
+ .filter{ dirName -> !excludedModules.contains(dirName) }
+ .filter{ dirName -> !executedBaseNames.contains('#' + dirName) }
.filter{ dirName -> !executedBaseNames.contains(dirName) }
.forEach{ dirName -> missingBaseNames.add(dirName) }