You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2023/02/01 11:31:42 UTC
[druid] branch master updated: Migrate ITs from Travis to GHA (#13681)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c95a26cae3 Migrate ITs from Travis to GHA (#13681)
c95a26cae3 is described below
commit c95a26cae3ecafca430326ae8302fefda59b9fbd
Author: Tejaswini Bandlamudi <96...@users.noreply.github.com>
AuthorDate: Wed Feb 1 17:01:29 2023 +0530
Migrate ITs from Travis to GHA (#13681)
---
.github/workflows/reusable-revised-its.yml | 73 ++++++++++++
.github/workflows/reusable-standard-its.yml | 86 ++++++++++++++
.../{unit-tests.yml => reusable-unit-tests.yml} | 80 ++++++-------
.github/workflows/revised-its.yml | 36 ++++++
.github/workflows/standard-its.yml | 115 ++++++++++++++++++
.github/workflows/static-checks.yml | 1 +
.../unit-and-integration-tests-unified.yml | 130 +++++++++++++++++++++
.github/workflows/unit-tests.yml | 129 +++++++-------------
.../docker/docker-compose.query-error-test.yml | 2 +-
.../docker/docker-compose.query-retry-test.yml | 2 +-
.../apache/druid/tests/query/ITJdbcQueryTest.java | 2 +
it.sh | 12 ++
12 files changed, 536 insertions(+), 132 deletions(-)
diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml
new file mode 100644
index 0000000000..53a708add1
--- /dev/null
+++ b/.github/workflows/reusable-revised-its.yml
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Revised Integration Tests shared workflow
+on:
+ workflow_call:
+ inputs:
+ script:
+ description: 'Which IT script to run'
+ required: true
+ type: string
+ build_jdk:
+ description: 'Which jdk version was used to build'
+ required: true
+ type: string
+ runtime_jdk:
+ description: 'Which JDK version to use at runtime'
+ required: true
+ type: string
+ use_indexer:
+ description: 'Which indexer to use'
+ required: true
+ type: string
+ it:
+ description: 'IT test Category'
+ required: true
+ type: string
+
+env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+ test: # Github job that runs a given revised/new IT against retrieved cached druid docker image
+ name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }})
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout branch
+ uses: actions/checkout@v3
+
+ - name: Setup java
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: ${{ inputs.build_jdk }}
+ cache: maven
+
+ - name: Retrieve cached docker image
+ uses: actions/cache/restore@v3
+ with:
+ key: druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+ path: |
+ ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+ ./integration-tests-ex/image/target/env.sh
+
+ - name: Load docker image
+ run: |
+ docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
+ docker images
+
+ - name: Run IT
+ run: ${{ inputs.script }}
diff --git a/.github/workflows/reusable-standard-its.yml b/.github/workflows/reusable-standard-its.yml
new file mode 100644
index 0000000000..a2467d6c27
--- /dev/null
+++ b/.github/workflows/reusable-standard-its.yml
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Revised Integration Tests shared workflow
+on:
+ workflow_call:
+ inputs:
+ runtime_jdk:
+ description: 'Which JDK version to use at runtime'
+ required: true
+ type: string
+ use_indexer:
+ description: 'Which indexer to use'
+ required: true
+ type: string
+ override_config_path:
+ description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage'
+ required: false
+ type: string
+ group:
+ required: true
+ type: string
+ description: 'Name of group of tests running (to display)'
+ testing_groups:
+ required: true
+ type: string
+ build_jdk:
+ description: 'Which jdk version was used to build'
+ required: true
+ type: string
+ mysql_driver:
+ description: 'MySQL driver to use'
+ required: false
+ type: string
+ default: com.mysql.jdbc.Driver
+
+env:
+ MVN: mvn --no-snapshot-updates
+ MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+ MAVEN_SKIP_TESTS: -P skip-tests
+ DOCKER_IP: 127.0.0.1 # for integration tests
+ MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+ test: # Github job that runs a given standard/old IT for a given testing group
+ name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout branch
+ uses: actions/checkout@v3
+
+ - name: Setup java
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: ${{ inputs.runtime_jdk }}
+
+ - name: Restore Maven repository
+ id: maven-restore
+ uses: actions/cache/restore@v3
+ with:
+ path: ~/.m2/repository
+ key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ hashFiles('**/pom.xml') }}
+
+ - name: Run IT
+ env:
+ MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
+ MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
+ run: |
+ # Debug echo
+ echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}"
+ echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
+ MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/reusable-unit-tests.yml
similarity index 51%
copy from .github/workflows/unit-tests.yml
copy to .github/workflows/reusable-unit-tests.yml
index 77e0663adc..1684a5fe57 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/reusable-unit-tests.yml
@@ -11,22 +11,29 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License
-name: (openjdk8) Unit Tests CI
+name: Revised Unit Tests shared workflow
on:
- push:
- branches:
- - master
- - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
- pull_request:
- branches:
- - master
- - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
-
-concurrency:
- group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
- cancel-in-progress: true
+ workflow_call:
+ inputs:
+ jdk:
+ required: true
+ type: string
+ description: 'JDK version used to test Druid'
+ sql_compatibility:
+ required: false
+ type: boolean
+ default: true
+ description: 'For SQL compatibility'
+ module:
+ required: true
+ type: string
+ description: 'Name of group of tests that are running'
+ maven_projects:
+ required: true
+ type: string
+ description: 'Name of group of tests running (to display)'
env:
MVN: mvn -B
@@ -34,12 +41,11 @@ env:
MAVEN_SKIP_TESTS: -P skip-tests
MAVEN_OPTS: -Xmx3000m
FORCE_COLOR: 2
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
unit-tests:
- strategy:
- matrix:
- DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
+ name: ${{ inputs.module }} modules test
runs-on: ubuntu-latest
steps:
- name: checkout branch
@@ -47,18 +53,18 @@ jobs:
with:
fetch-depth: 0
- - name: setup jdk8
+ - name: setup jdk${{ inputs.jdk }}
uses: actions/setup-java@v3
with:
distribution: 'zulu'
- java-version: 8
- cache: 'maven'
+ java-version: ${{ inputs.jdk }}
- - name: maven install
- run: |
- echo 'Running Maven install...' &&
- ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
- ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+ - name: Restore Maven repository
+ id: maven-restore
+ uses: actions/cache/restore@v3
+ with:
+ path: ~/.m2/repository
+ key: maven-${{ runner.os }}-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml') }}
- name: setup node
uses: actions/setup-node@v3
@@ -69,7 +75,7 @@ jobs:
run: |
export base_ref=${{ github.base_ref }}
echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
- export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
+ export druid_use_default_value_for_null=${{ inputs.sql_compatibility }}
echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
- name: fetch base branch for test coverage
@@ -79,25 +85,11 @@ jobs:
# This is not needed for build triggered by tags, since there will be no code diff.
git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
- - name: Setup diff-test-coverage
+ - name: setup diff-test-coverage
run: npm install @connectis/diff-test-coverage@1.5.3
- - name: "(openjdk8) indexing modules test"
- env:
- MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) processing module test"
- env:
- MAVEN_PROJECTS: processing
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) server module test"
- env:
- MAVEN_PROJECTS: server
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) other modules test"
+ - name: test & coverage
env:
- MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
+ MAVEN_PROJECTS: ${{ inputs.maven_projects }}
+ MVN: ${{ format('{0} {1}', env.MVN, (steps.maven-restore.outputs.cache-hit && 'dependency:go-offline' || '-U')) }}
run: ./.github/scripts/unit_tests_script.sh
diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml
new file mode 100644
index 0000000000..056fa584e3
--- /dev/null
+++ b/.github/workflows/revised-its.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Github workflow that runs revised/new ITs
+on:
+ workflow_call
+
+jobs:
+ it:
+ strategy:
+ fail-fast: false
+ matrix:
+ #jdk: [8, 11, 17]
+ jdk: [8]
+ it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat]
+ #indexer: [indexer, middleManager]
+ indexer: [middleManager]
+ uses: apache/druid/.github/workflows/reusable-revised-its.yml@master
+ with:
+ build_jdk: ${{ matrix.jdk }}
+ runtime_jdk: ${{ matrix.jdk }}
+ use_indexer: ${{ matrix.indexer }}
+ script: ./it.sh github ${{ matrix.it }}
+ it: ${{ matrix.it }}
diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml
new file mode 100644
index 0000000000..c0e3a0b6f9
--- /dev/null
+++ b/.github/workflows/standard-its.yml
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Github workflow that runs standard/old ITs
+on:
+ workflow_call
+
+jobs:
+ integration-index-tests-middleManager:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [8, 11]
+ testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: ${{ matrix.jdk }}
+ testing_groups: -Dgroups=${{ matrix.testing_group }}
+ use_indexer: middleManager
+ group: ${{ matrix.testing_group }}
+
+ integration-index-tests-indexer:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [8, 11]
+ testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: ${{ matrix.jdk }}
+ testing_groups: -Dgroups=${{ matrix.testing_group }}
+ use_indexer: indexer
+ group: ${{ matrix.testing_group }}
+
+ integration-query-tests-middleManager:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [8, 11]
+ testing_group: [query, query-retry, query-error, security, high-availability]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: ${{ matrix.jdk }}
+ testing_groups: -Dgroups=${{ matrix.testing_group }}
+ use_indexer: middleManager
+ override_config_path: ./environment-configs/test-groups/prepopulated-data
+ group: ${{ matrix.testing_group }}
+
+ integration-query-tests-middleManager-mariaDB:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [8, 11]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: ${{ matrix.jdk }}
+ testing_groups: -Dgroups=query
+ use_indexer: middleManager
+ mysql_driver: org.mariadb.jdbc.Driver
+ override_config_path: ./environment-configs/test-groups/prepopulated-data
+ group: query
+
+ integration-shuffle-deep-store-tests:
+ strategy:
+ fail-fast: false
+ matrix:
+ indexer: [indexer, middleManager]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: 8
+ testing_groups: -Dgroups=shuffle-deep-store
+ use_indexer: ${{ matrix.indexer }}
+ override_config_path: ./environment-configs/test-groups/shuffle-deep-store
+ group: shuffle deep store
+
+ integration-custom-coordinator-duties-tests:
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: 8
+ testing_groups: -Dgroups=custom-coordinator-duties
+ use_indexer: middleManager
+ override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
+ group: custom coordinator duties
+
+ integration-other-tests:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [8, 11]
+ indexer: [middleManager, indexer]
+ uses: apache/druid/.github/workflows/reusable-standard-its.yml@master
+ with:
+ build_jdk: 8
+ runtime_jdk: ${{ matrix.jdk }}
+ testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-sto [...]
+ use_indexer: ${{ matrix.indexer }}
+ group: other
diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml
index f247bf01a1..a0baecdd0e 100644
--- a/.github/workflows/static-checks.yml
+++ b/.github/workflows/static-checks.yml
@@ -37,6 +37,7 @@ env:
jobs:
static-checks:
strategy:
+ fail-fast: false
matrix:
java: [ 'jdk8', 'jdk11', 'jdk17' ]
runs-on: ubuntu-latest
diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml
new file mode 100644
index 0000000000..561ed0f0a4
--- /dev/null
+++ b/.github/workflows/unit-and-integration-tests-unified.yml
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Unit & Integration tests CI
+on:
+ push:
+ branches:
+ - master
+ - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+ pull_request:
+ branches:
+ - master
+ - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+ cancel-in-progress: true
+
+env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+ build:
+ name: build (jdk${{ matrix.jdk }})
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [ '8', '11', '17' ]
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout branch
+ uses: actions/checkout@v3
+
+ - name: Cache Maven m2 repository
+ id: maven
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2/repository
+ key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-${{ runner.os }}-${{ matrix.jdk }}
+
+ - name: Setup java
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: ${{ matrix.jdk }}
+
+ - name: Maven build
+ id: maven_build
+ run: |
+ ./it.sh ci
+
+ - name: Cache targets
+ id: target
+ uses: actions/cache@v3
+ with:
+ path: |
+ ./**/target
+ key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-${{ runner.os }}-${{ matrix.jdk }}-targets
+
+ - name: Container build
+ run: |
+ ./it.sh image
+ source ./integration-tests-ex/image/target/env.sh
+ docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}
+
+ - name: Save docker container to archive
+ run: |
+ source ./integration-tests-ex/image/target/env.sh
+ echo $DRUID_IT_IMAGE_NAME
+ docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz
+
+ - name: Cache image
+ id: docker_container
+ uses: actions/cache@v3
+ with:
+ key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ hashFiles('**/pom.xml') }}
+ path: |
+ ./druid-container-jdk${{ matrix.jdk }}.tar.gz
+ ./integration-tests-ex/image/target/env.sh
+ restore-keys: |
+ druid-container-jdk${{ matrix.jdk }}.tar.gz-
+
+ unit-tests:
+ strategy:
+ fail-fast: false
+ matrix:
+ sql_compatibility: [ false, true ]
+ name: unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})
+ uses: apache/druid/.github/workflows/unit-tests.yml@master
+ needs: build
+ with:
+ jdk: 8
+ sql_compatibility: ${{ matrix.sql_compatibility }}
+
+ unit-tests-phase2:
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [11, 17]
+ sql_compatibility: [ false, true ]
+ name: unit tests (jdk${{ matrix.jdk }}, sql-compat=${{ matrix.sql_compatibility }})
+ uses: apache/druid/.github/workflows/unit-tests.yml@master
+ needs: unit-tests
+ with:
+ jdk: ${{ matrix.jdk }}
+ sql_compatibility: ${{ matrix.sql_compatibility }}
+
+ standard-its:
+ needs: unit-tests
+ uses: apache/druid/.github/workflows/standard-its.yml@master
+
+ revised-its:
+ needs: unit-tests
+ uses: apache/druid/.github/workflows/revised-its.yml@master
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 77e0663adc..ef500aa2ef 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -11,93 +11,50 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License
-name: (openjdk8) Unit Tests CI
on:
- push:
- branches:
- - master
- - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
- pull_request:
- branches:
- - master
- - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
-
-concurrency:
- group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
- cancel-in-progress: true
-
-env:
- MVN: mvn -B
- MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
- MAVEN_SKIP_TESTS: -P skip-tests
- MAVEN_OPTS: -Xmx3000m
- FORCE_COLOR: 2
+ workflow_call:
+ inputs:
+ jdk:
+ required: true
+ type: string
+ description: 'JDK version used to test Druid'
+ sql_compatibility:
+ required: false
+ type: boolean
+ default: true
+ description: 'For SQL compatibility'
jobs:
- unit-tests:
- strategy:
- matrix:
- DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
- runs-on: ubuntu-latest
- steps:
- - name: checkout branch
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
-
- - name: setup jdk8
- uses: actions/setup-java@v3
- with:
- distribution: 'zulu'
- java-version: 8
- cache: 'maven'
-
- - name: maven install
- run: |
- echo 'Running Maven install...' &&
- ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
- ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
-
- - name: setup node
- uses: actions/setup-node@v3
- with:
- node-version: 16.17.0
-
- - name: setup variables
- run: |
- export base_ref=${{ github.base_ref }}
- echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
- export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
- echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
-
- - name: fetch base branch for test coverage
- if: ${{ github.base_ref != '' }}
- run: |
- # Add merge target branch to determine diff.
- # This is not needed for build triggered by tags, since there will be no code diff.
- git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
-
- - name: Setup diff-test-coverage
- run: npm install @connectis/diff-test-coverage@1.5.3
-
- - name: "(openjdk8) indexing modules test"
- env:
- MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) processing module test"
- env:
- MAVEN_PROJECTS: processing
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) server module test"
- env:
- MAVEN_PROJECTS: server
- run: ./.github/scripts/unit_tests_script.sh
-
- - name: "(openjdk8) other modules test"
- env:
- MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
- run: ./.github/scripts/unit_tests_script.sh
+ indexing_modules_test:
+ uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
+ with:
+ jdk: ${{ inputs.jdk }}
+ sql_compatibility: ${{ inputs.sql_compatibility }}
+ module: indexing
+ maven_projects: 'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
+
+ processing_modules_test:
+ uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
+ with:
+ jdk: ${{ inputs.jdk }}
+ sql_compatibility: ${{ inputs.sql_compatibility }}
+ module: processing
+ maven_projects: 'processing'
+
+ server_modules_test:
+ uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
+ with:
+ jdk: ${{ inputs.jdk }}
+ sql_compatibility: ${{ inputs.sql_compatibility }}
+ module: server
+ maven_projects: 'server'
+
+ other_modules_test:
+ uses: apache/druid/.github/workflows/reusable-unit-tests.yml@master
+ with:
+ jdk: ${{ inputs.jdk }}
+ sql_compatibility: ${{ inputs.sql_compatibility }}
+ module: other
+ maven_projects: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
diff --git a/integration-tests/docker/docker-compose.query-error-test.yml b/integration-tests/docker/docker-compose.query-error-test.yml
index 5dc175829a..f75ff2efad 100644
--- a/integration-tests/docker/docker-compose.query-error-test.yml
+++ b/integration-tests/docker/docker-compose.query-error-test.yml
@@ -78,7 +78,7 @@ services:
druid-it-net:
ipv4_address: 172.172.172.14
ports:
- - 8084:8083
+ - 8086:8083 #8084 is used by mono service on GHA runners
- 8284:8283
- 5010:5007
privileged: true
diff --git a/integration-tests/docker/docker-compose.query-retry-test.yml b/integration-tests/docker/docker-compose.query-retry-test.yml
index adba343557..2efb859a6f 100644
--- a/integration-tests/docker/docker-compose.query-retry-test.yml
+++ b/integration-tests/docker/docker-compose.query-retry-test.yml
@@ -78,7 +78,7 @@ services:
druid-it-net:
ipv4_address: 172.172.172.14
ports:
- - 8084:8083
+ - 8086:8083 #8084 is used by mono service on GHA runners
- 8284:8283
- 5010:5007
privileged: true
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
index fe95c27bc7..72e682c9fb 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
@@ -108,6 +108,8 @@ public class ITJdbcQueryTest
};
// ensure that wikipedia segments are loaded completely
dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
+ dataLoaderHelper.waitUntilDatasourceIsReady("wikipedia");
+ dataLoaderHelper.waitUntilDatasourceIsReady("twitterstream");
}
@Test
diff --git a/it.sh b/it.sh
index 9988f6b5c8..7fb3889b55 100755
--- a/it.sh
+++ b/it.sh
@@ -31,6 +31,8 @@ function usage
{
cat <<EOF
Usage: $0 cmd [category]
+ ci
+ build Druid and the distribution for CI pipelines
build
build Druid and the distribution
dist
@@ -49,6 +51,8 @@ Usage: $0 cmd [category]
show the last 20 lines of each container log
travis <category>
run one IT in Travis (build dist, image, run test, tail logs)
+ github <category>
+ run one IT in Github Workflows (run test, tail logs)
prune
prune Docker volumes
@@ -189,6 +193,9 @@ case $CMD in
"help" )
usage
;;
+ "ci" )
+ mvn -q clean package dependency:go-offline -P dist $MAVEN_IGNORE -T1.0C
+ ;;
"build" )
mvn clean package -P dist $MAVEN_IGNORE -T1.0C
;;
@@ -230,6 +237,11 @@ case $CMD in
$0 test $CATEGORY
$0 tail $CATEGORY
;;
+ "github" )
+ prepare_category $1
+ $0 test $CATEGORY
+ $0 tail $CATEGORY
+ ;;
"prune" )
# Caution: this removes all volumes, which is generally what you
# want when testing.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org