You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2022/06/05 10:13:01 UTC
[pulsar] branch master updated: [CI] Run pulsar-io unit tests in separate build job (#15926)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 813aeccd7dd [CI] Run pulsar-io unit tests in separate build job (#15926)
813aeccd7dd is described below
commit 813aeccd7dd28820573156d089544096c57d0855
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Sun Jun 5 13:12:50 2022 +0300
[CI] Run pulsar-io unit tests in separate build job (#15926)
* [CI] Run pulsar-io unit tests separately
* Move pulsar-sql tests to run with PULSAR_IO unit test group
* Use "clean install" to resolve issues with shading
* Exclude distributions from other test group
- distributions don't contain unit tests so no need to include in unit test run
* Skip integration tests
---
.github/workflows/pulsar-ci.yaml | 2 ++
build/run_unit_group.sh | 31 +++++++++++++++++++++++--------
pom.xml | 14 ++++++++++++++
pulsar-broker/pom.xml | 2 +-
pulsar-io/pom.xml | 37 +++++++++++++++++++++++++++++++++++++
pulsar-proxy/pom.xml | 14 ++++++++++++++
pulsar-sql/pom.xml | 32 +++++++++++++++++++++++++-------
7 files changed, 116 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 3437d4f3700..a551b836db8 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -159,6 +159,8 @@ jobs:
timeout: 75
- name: Proxy
group: PROXY
+ - name: Pulsar IO
+ group: PULSAR_IO
steps:
- name: checkout
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index 2431c6305d8..49d5224f9db 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -34,6 +34,11 @@ function mvn_test() {
RETRY=""
shift
fi
+ local clean_arg=""
+ if [[ "$1" == "--clean" ]]; then
+ clean_arg="clean"
+ shift
+ fi
TARGET=verify
if [[ "$1" == "--install" ]]; then
TARGET="install"
@@ -42,7 +47,7 @@ function mvn_test() {
echo "::group::Run tests for " "$@"
# use "verify" instead of "test" to workaround MDEP-187 issue in pulsar-functions-worker and pulsar-broker projects with the maven-dependency-plugin's copy goal
# Error message was "Artifact has not been packaged yet. When used on reactor artifact, copy should be executed after packaging: see MDEP-187"
- $RETRY $MVN_TEST_OPTIONS $TARGET "$@" "${COMMANDLINE_ARGS[@]}"
+ $RETRY $MVN_TEST_OPTIONS $clean_arg $TARGET "$@" "${COMMANDLINE_ARGS[@]}"
echo "::endgroup::"
set +x
"$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
@@ -129,25 +134,24 @@ function test_group_proxy() {
}
function test_group_other() {
- mvn_test --install -PbrokerSkipTest \
- -Dexclude='org/apache/pulsar/proxy/**/*.java,
- **/ManagedLedgerTest.java,
- **/TestPulsarKeyValueSchemaHandler.java,
+ mvn_test --clean --install \
+ -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution' \
+ -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \
+ -Dexclude='**/ManagedLedgerTest.java,
+ **/OffloadersCacheTest.java
**/PrimitiveSchemaTest.java,
BlobStoreManagedLedgerOffloaderTest.java'
mvn_test -pl managed-ledger -Dinclude='**/ManagedLedgerTest.java,
**/OffloadersCacheTest.java'
- mvn_test -pl pulsar-sql/presto-pulsar-plugin -Dinclude='**/TestPulsarKeyValueSchemaHandler.java'
-
mvn_test -pl pulsar-client -Dinclude='**/PrimitiveSchemaTest.java'
mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java'
echo "::endgroup::"
local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \
- awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy' | sort | uniq | \
+ awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-sql' | sort | uniq | \
perl -0777 -p -e 's/\n(\S)/,$1/g')
if [ -n "${modules_with_quarantined_tests}" ]; then
echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)"
@@ -158,6 +162,17 @@ function test_group_other() {
fi
}
+function test_group_pulsar_io() {
+ $MVN_TEST_OPTIONS -pl kafka-connect-avro-converter-shaded clean install
+ echo "::group::Running pulsar-io tests"
+ mvn_test --install -Ppulsar-io-tests,-main
+ echo "::endgroup::"
+
+ echo "::group::Running pulsar-sql tests"
+ mvn_test --install -Ppulsar-sql-tests,-main
+ echo "::endgroup::"
+}
+
function list_test_groups() {
declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
}
diff --git a/pom.xml b/pom.xml
index 837648798cc..2176fcf2f74 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2353,6 +2353,20 @@ flexible messaging model and an intuitive client API.</description>
<os.detected.classifier>osx-x86_64</os.detected.classifier>
</properties>
</profile>
+
+ <profile>
+ <id>pulsar-io-tests</id>
+ <modules>
+ <module>pulsar-io</module>
+ </modules>
+ </profile>
+
+ <profile>
+ <id>pulsar-sql-tests</id>
+ <modules>
+ <module>pulsar-sql</module>
+ </modules>
+ </profile>
</profiles>
<repositories>
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 9ea564151df..c6d55b5f5f9 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -773,7 +773,7 @@
</build>
</profile>
<profile>
- <id>brokerSkipTest</id>
+ <id>skipTestsForUnitGroupOther</id>
<build>
<plugins>
<plugin>
diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml
index 21e1b4c407d..b2c400117a5 100644
--- a/pulsar-io/pom.xml
+++ b/pulsar-io/pom.xml
@@ -76,6 +76,43 @@
</modules>
</profile>
+ <profile>
+ <id>pulsar-io-tests</id>
+ <modules>
+ <module>core</module>
+ <module>batch-discovery-triggerers</module>
+ <module>batch-data-generator</module>
+ <module>common</module>
+ <module>docs</module>
+ <module>aws</module>
+ <module>twitter</module>
+ <module>cassandra</module>
+ <module>aerospike</module>
+ <module>kafka</module>
+ <module>rabbitmq</module>
+ <module>kinesis</module>
+ <module>hdfs3</module>
+ <module>jdbc</module>
+ <module>data-generator</module>
+ <module>elastic-search</module>
+ <module>kafka-connect-adaptor</module>
+ <module>kafka-connect-adaptor-nar</module>
+ <module>debezium</module>
+ <module>hdfs2</module>
+ <module>canal</module>
+ <module>file</module>
+ <module>netty</module>
+ <module>hbase</module>
+ <module>mongo</module>
+ <module>flume</module>
+ <module>redis</module>
+ <module>solr</module>
+ <module>influxdb</module>
+ <module>dynamodb</module>
+ <module>nsq</module>
+ </modules>
+ </profile>
+
<profile>
<id>core-modules</id>
<modules>
diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml
index 18d64f105a9..7a80ec83f43 100644
--- a/pulsar-proxy/pom.xml
+++ b/pulsar-proxy/pom.xml
@@ -262,6 +262,20 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>skipTestsForUnitGroupOther</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml
index 400c7c5b0a4..71b951b5b5e 100644
--- a/pulsar-sql/pom.xml
+++ b/pulsar-sql/pom.xml
@@ -31,13 +31,6 @@
<artifactId>pulsar-sql</artifactId>
<name>Pulsar SQL :: Parent</name>
- <modules>
- <module>presto-pulsar</module>
- <module>presto-pulsar-plugin</module>
- <module>java-version-trim-agent</module>
- <module>presto-distribution</module>
- </modules>
-
<properties>
<!-- keep using okhttp3 3.x for Presto -->
<okhttp3.version>3.14.9</okhttp3.version>
@@ -178,6 +171,31 @@
</build>
<profiles>
+ <profile>
+ <id>main</id>
+ <activation>
+ <property>
+ <name>disableSqlMainProfile</name>
+ <!-- always active unless true is passed as a value -->
+ <value>!true</value>
+ </property>
+ </activation>
+ <modules>
+ <module>presto-pulsar</module>
+ <module>presto-pulsar-plugin</module>
+ <module>java-version-trim-agent</module>
+ <module>presto-distribution</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>pulsar-sql-tests</id>
+ <modules>
+ <module>presto-pulsar</module>
+ <module>presto-pulsar-plugin</module>
+ <module>java-version-trim-agent</module>
+ <module>presto-distribution</module>
+ </modules>
+ </profile>
<!--
The only working way for OWASP dependency checker plugin
to exclude module when failBuildOnCVSS is used