You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ol...@apache.org on 2026/10/18 10:35:35 UTC
(maven-surefire) branch master updated: Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)
This is an automated email from the ASF dual-hosted git repository.
olamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/master by this push:
new 907a2e216 Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)
907a2e216 is described below
commit 907a2e21646744b5f4707e4b0b4a05b4358e9c83
Author: Olivier Lamy <ol...@apache.org>
AuthorDate: Sun Oct 18 20:35:28 2026 +1000
Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)
Since 3.6.0, having TestNG on the test classpath turned off group
filtering for JUnit 5 and JUnit 4 tests, so groups only reached TestNG.
Signed-off-by: Olivier Lamy <ol...@apache.org>
---
.../src/site/markdown/architecture.md | 11 +-
.../its/GroupsWithTestNgOnClasspathIT.java | 109 ++++++++++++++++++
.../groups-with-testng-and-jupiter-tests/pom.xml | 87 ++++++++++++++
.../src/test/java/mixed/FastJupiterTest.java | 32 ++++++
.../src/test/java/mixed/FastTestNgTest.java | 30 +++++
.../src/test/java/mixed/SlowJupiterTest.java | 32 ++++++
.../src/test/java/mixed/SlowTestNgTest.java | 30 +++++
.../groups-with-testng-on-classpath/pom.xml | 79 +++++++++++++
.../src/test/java/jupiteronly/Base.java | 34 ++++++
.../src/test/java/jupiteronly/BaseTest.java | 31 +++++
.../test/java/jupiteronly/LifecycleSupport.java | 33 ++++++
.../src/test/java/jupiteronly/OtherTest.java | 30 +++++
.../junitplatform/JUnitPlatformProvider.java | 125 ++++++++++++++++-----
.../JUnitPlatformProviderGroupsTest.java | 108 ++++++++++++++++++
14 files changed, 736 insertions(+), 35 deletions(-)
diff --git a/maven-surefire-plugin/src/site/markdown/architecture.md b/maven-surefire-plugin/src/site/markdown/architecture.md
index e1edaf09a..4a8fe9205 100644
--- a/maven-surefire-plugin/src/site/markdown/architecture.md
+++ b/maven-surefire-plugin/src/site/markdown/architecture.md
@@ -439,16 +439,17 @@ The `maven-surefire-report-plugin` reads the XML files (via `surefire-report-par
## Group / Category Filtering
-Since 3.6.0, test filtering by groups, categories, or tags is handled via the JUnit Platform's native **tag expression** syntax.
+Since 3.6.0, test filtering by groups, categories, or tags is handled by the JUnit Platform provider, and each kind of test is matched in its own way.
### How it works
1. Users configure `<groups>` and `<excludedGroups>` in the plugin configuration
-2. For **JUnit 4**: `@Category` annotations are mapped to JUnit Platform tags via the Vintage Engine
-3. For **TestNG**: groups are mapped to JUnit Platform tags via the TestNG Engine
-4. For **JUnit 5/6**: `@Tag` annotations are used natively
+2. For **JUnit 4**: tests are matched against their `@Category` annotations, by fully qualified or simple class name
+3. For **TestNG**: the groups are handed to the TestNG Engine as `testng.groups` and `testng.excludedGroups`, and TestNG matches them itself, regular expressions included
+4. For **JUnit 5/6** and all other engines: `@Tag` annotations are matched with the JUnit Platform's native **tag expression** syntax
-Tag filter expressions are passed directly to the JUnit Platform Launcher.
+This does not change when TestNG is on the test classpath: the tests of the other engines are still filtered.
+A group which TestNG accepts but JUnit cannot parse as a tag expression, like `(?!slow).*`, only filters TestNG tests, and Surefire logs a warning about it.
---
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java
new file mode 100644
index 000000000..aa6bec2f4
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Checks that groups and excludedGroups still filter Jupiter tests when TestNG is on the classpath,
+ * and that TestNG still filters its own tests.
+ */
+public class GroupsWithTestNgOnClasspathIT extends SurefireJUnit4IntegrationTestCase {
+
+ @Test
+ void groupsOnlyRunTaggedJupiterTestsWhenTestNgIsOnTheClasspath() {
+ OutputValidator outputValidator = unpack("groups-with-testng-on-classpath")
+ .setGroups("base")
+ .executeTest()
+ .verifyTextInLog("TestNG is present")
+ .verifyErrorFree(1);
+
+ assertReport(outputValidator, "jupiteronly.BaseTest", true);
+ assertReport(outputValidator, "jupiteronly.OtherTest", false);
+ }
+
+ @Test
+ void excludedGroupsSkipTaggedJupiterTestsWhenTestNgIsOnTheClasspath() {
+ OutputValidator outputValidator = unpack("groups-with-testng-on-classpath")
+ .setExcludedGroups("base")
+ .executeTest()
+ .verifyTextInLog("TestNG is present")
+ .verifyErrorFree(1);
+
+ assertReport(outputValidator, "jupiteronly.BaseTest", false);
+ assertReport(outputValidator, "jupiteronly.OtherTest", true);
+ }
+
+ @Test
+ void groupsFilterTestNgAndJupiterTests() {
+ OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+ .setGroups("fast")
+ .executeTest()
+ .verifyErrorFree(2);
+
+ assertOnlyFastTestsRan(outputValidator);
+ }
+
+ @Test
+ void excludedGroupsFilterTestNgAndJupiterTests() {
+ OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+ .setExcludedGroups("slow")
+ .executeTest()
+ .verifyErrorFree(2);
+
+ assertOnlyFastTestsRan(outputValidator);
+ }
+
+ @Test
+ void groupsWhichAreNotTagExpressionsOnlyFilterTestNgTests() {
+ // (?!slow).* works for TestNG, but JUnit can't parse it as a tag expression
+ OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+ .activateProfile("testng-regex")
+ .executeTest()
+ .verifyTextInLog(
+ "These groups are not valid JUnit tag expressions, so they only filter TestNG tests: (?!slow).*")
+ .verifyErrorFree(3);
+
+ assertReport(outputValidator, "mixed.FastTestNgTest", true);
+ assertReport(outputValidator, "mixed.SlowTestNgTest", false);
+ assertReport(outputValidator, "mixed.FastJupiterTest", true);
+ assertReport(outputValidator, "mixed.SlowJupiterTest", true);
+ }
+
+ private static void assertOnlyFastTestsRan(OutputValidator outputValidator) {
+ assertReport(outputValidator, "mixed.FastTestNgTest", true);
+ assertReport(outputValidator, "mixed.SlowTestNgTest", false);
+ assertReport(outputValidator, "mixed.FastJupiterTest", true);
+ assertReport(outputValidator, "mixed.SlowJupiterTest", false);
+ }
+
+ private static void assertReport(OutputValidator outputValidator, String testClass, boolean ran) {
+ if (ran) {
+ outputValidator
+ .getSurefireReportsXmlFile("TEST-" + testClass + ".xml")
+ .assertFileExists();
+ } else {
+ outputValidator
+ .getSurefireReportsXmlFile("TEST-" + testClass + ".xml")
+ .assertFileNotExists();
+ }
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml
new file mode 100644
index 000000000..3b4f38fcc
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>groups-with-testng-and-jupiter-tests</artifactId>
+ <version>1.0</version>
+ <name>Groups with TestNG and Jupiter tests side by side</name>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <version>5.11.4</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- Surefire adds the testng-engine on its own -->
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>7.5.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.0</version>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>testng-regex</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- works for TestNG, but JUnit can't parse it as a tag expression -->
+ <groups>(?!slow).*</groups>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java
new file mode 100644
index 000000000..4639b9b86
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("fast")
+class FastJupiterTest {
+ @Test
+ void test() {
+ assertEquals(2, 1 + 1);
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java
new file mode 100644
index 000000000..65e888bf1
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class FastTestNgTest {
+ @Test(groups = "fast")
+ public void test() {
+ assertEquals(1 + 1, 2);
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java
new file mode 100644
index 000000000..a34e8fe2f
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("slow")
+class SlowJupiterTest {
+ @Test
+ void test() {
+ assertEquals(2, 1 + 1);
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java
new file mode 100644
index 000000000..4c4076563
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class SlowTestNgTest {
+ @Test(groups = "slow")
+ public void test() {
+ assertEquals(1 + 1, 2);
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml b/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml
new file mode 100644
index 000000000..cc39db279
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>groups-with-testng-on-classpath</artifactId>
+ <version>1.0</version>
+ <name>Jupiter tags with TestNG and JUnit 4 on the test classpath</name>
+ <description>
+ Only Jupiter tests here. JUnit 4 and TestNG are only needed by a helper class, like InjectedTest in Eclipse Sisu.
+ </description>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>5.14.4</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- only used by the helper class, there are no JUnit 4 or TestNG tests -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.14.3</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.0</version>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java
new file mode 100644
index 000000000..62147da7c
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Tag;
+
+/**
+ * Adds the base tag to a test class.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("base")
+public @interface Base {}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java
new file mode 100644
index 000000000..211bb897c
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Base
+class BaseTest extends LifecycleSupport {
+ @Test
+ void test() {
+ assertTrue(ready);
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java
new file mode 100644
index 000000000..c0feb47cd
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+/**
+ * Helper that works with JUnit 4, JUnit 5 and TestNG, which is why all three are on the classpath.
+ */
+public abstract class LifecycleSupport {
+ protected boolean ready;
+
+ @org.junit.Before
+ @org.junit.jupiter.api.BeforeEach
+ @org.testng.annotations.BeforeMethod
+ public void setUp() {
+ ready = true;
+ }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java
new file mode 100644
index 000000000..d8b512051
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class OtherTest extends LifecycleSupport {
+ @Test
+ void test() {
+ assertTrue(ready);
+ }
+}
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
index 66481963f..6f4dc15bc 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -56,6 +56,7 @@
import org.apache.maven.surefire.api.util.ScanResult;
import org.apache.maven.surefire.api.util.TestsToRun;
import org.apache.maven.surefire.shared.utils.StringUtils;
+import org.junit.platform.commons.JUnitException;
import org.junit.platform.engine.ConfigurationParameters;
import org.junit.platform.engine.DiscoverySelector;
import org.junit.platform.engine.Filter;
@@ -128,6 +129,9 @@ public class JUnitPlatformProvider extends AbstractProvider {
private boolean excludeTestNgEngine;
+ // groups only TestNG can use, we warn about them when the tests start
+ private final List<String> testNgOnlyGroups = new ArrayList<>();
+
private final CommandChainReader commandsReader;
public JUnitPlatformProvider(ProviderParameters parameters) {
@@ -225,6 +229,7 @@ public RunResult invoke(Object forkTestSet) throws TestSetFailedException, Repor
ReporterFactory reporterFactory = parameters.getReporterFactory();
final RunResult runResult;
TestReportListener<TestOutputReportEntry> runListener = reporterFactory.createTestReportListener();
+ warnAboutTestNgOnlyGroups(runListener);
CancellationTokenAdapter cancellationToken = CancellationTokenAdapter.tryCreate();
Stoppable stoppable = prepareFailFastSupport(cancellationToken, runListener);
RunListenerAdapter adapter = new RunListenerAdapter(runListener, stoppable);
@@ -253,6 +258,13 @@ public RunResult invoke(Object forkTestSet) throws TestSetFailedException, Repor
return runResult;
}
+ void warnAboutTestNgOnlyGroups(ConsoleLogger logger) {
+ if (!testNgOnlyGroups.isEmpty()) {
+ logger.warning("These groups are not valid JUnit tag expressions, so they only filter TestNG tests: "
+ + String.join(", ", testNgOnlyGroups));
+ }
+ }
+
private static void setupJunitLogger() {
Logger logger = Logger.getLogger("org.junit");
if (logger.getLevel() == null) {
@@ -498,33 +510,8 @@ private Filter<?>[] newFilters() {
filters.add(includeEnclosingClasses(classNameFilter));
}
- boolean useTestNG = parameters.getProviderProperties().get("testng.version") != null;
+ filters.addAll(newGroupFilters());
- if (!Boolean.parseBoolean(parameters.getProviderProperties().get(JUNIT_VINTAGE_DETECTED)) && !useTestNG) {
- getPropertiesList(GROUPS_PROP).map(TagFilter::includeTags).ifPresent(filters::add);
- getPropertiesList(EXCLUDEDGROUPS_PROP).map(TagFilter::excludeTags).ifPresent(filters::add);
- Optional<Class<?>> categoryClass = getCategoryClass();
- if (categoryClass.isPresent()) {
- getPropertiesList(EXCLUDEDGROUPS_PROP)
- .map(strings -> byEngine(
- getExcludeCategoryFilter(strings, categoryClass),
- descriptor -> FilterResult.included("Not a JUnit 4 test")))
- .ifPresent(filters::add);
- }
- } else if (!useTestNG) {
- // JUnit 4 tests use @Category, all other tests use tags (#3468, #3503)
- Optional<Class<?>> categoryClass = getCategoryClass();
- if (categoryClass.isPresent()) {
- getPropertiesList(GROUPS_PROP)
- .map(strings -> byEngine(
- getIncludeCategoryFilter(strings, categoryClass), TagFilter.includeTags(strings)))
- .ifPresent(filters::add);
- getPropertiesList(EXCLUDEDGROUPS_PROP)
- .map(strings -> byEngine(
- getExcludeCategoryFilter(strings, categoryClass), TagFilter.excludeTags(strings)))
- .ifPresent(filters::add);
- }
- }
of(optionallyWildcardFilter(parameters.getTestRequest().getTestListResolver()))
.filter(f -> !f.isEmpty())
.filter(f -> !f.isWildcard())
@@ -542,6 +529,74 @@ private Filter<?>[] newFilters() {
return filters.toArray(new Filter<?>[0]);
}
+ /**
+ * Builds the filters for groups and excludedGroups. TestNG tests are left alone, TestNG gets the groups too
+ * and filters its own tests.
+ */
+ private List<PostDiscoveryFilter> newGroupFilters() {
+ Optional<PostDiscoveryFilter> includeTags =
+ getTagExpressions(GROUPS_PROP).map(TagFilter::includeTags);
+ Optional<PostDiscoveryFilter> excludeTags =
+ getTagExpressions(EXCLUDEDGROUPS_PROP).map(TagFilter::excludeTags);
+ Optional<Class<?>> categoryClass = getCategoryClass();
+ List<PostDiscoveryFilter> groupFilters = new ArrayList<>();
+ if (!Boolean.parseBoolean(parameters.getProviderProperties().get(JUNIT_VINTAGE_DETECTED))) {
+ includeTags.ifPresent(groupFilters::add);
+ excludeTags.ifPresent(groupFilters::add);
+ if (categoryClass.isPresent()) {
+ getPropertiesList(EXCLUDEDGROUPS_PROP)
+ .map(strings -> byEngine(
+ getExcludeCategoryFilter(strings, categoryClass),
+ descriptor -> FilterResult.included("Not a JUnit 4 test")))
+ .ifPresent(groupFilters::add);
+ }
+ } else if (categoryClass.isPresent()) {
+ // JUnit 4 tests use @Category, all other tests use tags (#3468, #3503)
+ PostDiscoveryFilter noTagExpression = descriptor -> FilterResult.included("No tag expression to match");
+ getPropertiesList(GROUPS_PROP)
+ .map(strings -> byEngine(
+ getIncludeCategoryFilter(strings, categoryClass), includeTags.orElse(noTagExpression)))
+ .ifPresent(groupFilters::add);
+ getPropertiesList(EXCLUDEDGROUPS_PROP)
+ .map(strings -> byEngine(
+ getExcludeCategoryFilter(strings, categoryClass), excludeTags.orElse(noTagExpression)))
+ .ifPresent(groupFilters::add);
+ }
+ return groupFilters.stream()
+ .map(JUnitPlatformProvider::leaveTestNgTestsAlone)
+ .collect(toList());
+ }
+
+ /**
+ * Returns the groups JUnit can parse as tag expressions. With TestNG, a group can also be a regular expression
+ * JUnit can't parse, like {@code (?!slow).*}. We leave those to TestNG instead of failing the run.
+ */
+ private Optional<List<String>> getTagExpressions(String key) {
+ Optional<List<String>> groups = getPropertiesList(key);
+ if (!groups.isPresent() || parameters.getProviderProperties().get("testng.version") == null) {
+ return groups;
+ }
+ List<String> tagExpressions = new ArrayList<>();
+ for (String group : groups.get()) {
+ if (isTagExpression(group)) {
+ tagExpressions.add(group);
+ } else {
+ testNgOnlyGroups.add(group);
+ }
+ }
+ return tagExpressions.isEmpty() ? empty() : of(tagExpressions);
+ }
+
+ private static boolean isTagExpression(String group) {
+ try {
+ TagFilter.includeTags(group);
+ return true;
+ } catch (JUnitException e) {
+ // not PreconditionViolationException, JUnit 5.4 and older don't have it
+ return false;
+ }
+ }
+
/**
* Gives back the names of the classes that enclose a scanned class. They are only looked up the first time
* someone asks, because most runs never need them.
@@ -609,15 +664,25 @@ PostDiscoveryFilter getExcludeCategoryFilter(List<String> categories, Optional<C
* Uses the first filter for JUnit 4 tests and the second one for everything else.
*/
private static PostDiscoveryFilter byEngine(PostDiscoveryFilter vintageFilter, PostDiscoveryFilter otherFilter) {
- return testDescriptor ->
- isVintage(testDescriptor) ? vintageFilter.apply(testDescriptor) : otherFilter.apply(testDescriptor);
+ return testDescriptor -> isFromEngine(testDescriptor, VINTAGE_ENGINE_ID)
+ ? vintageFilter.apply(testDescriptor)
+ : otherFilter.apply(testDescriptor);
+ }
+
+ /**
+ * TestNG tests always pass here, TestNG checks their groups itself.
+ */
+ private static PostDiscoveryFilter leaveTestNgTestsAlone(PostDiscoveryFilter filter) {
+ return testDescriptor -> isFromEngine(testDescriptor, TestNgTestClassDetector.TESTNG_ENGINE_ID)
+ ? FilterResult.included("TestNG matches the groups itself")
+ : filter.apply(testDescriptor);
}
- private static boolean isVintage(TestDescriptor testDescriptor) {
+ private static boolean isFromEngine(TestDescriptor testDescriptor, String engineId) {
return testDescriptor
.getUniqueId()
.getEngineId()
- .filter(VINTAGE_ENGINE_ID::equals)
+ .filter(engineId::equals)
.isPresent();
}
diff --git a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
index c70c1806d..774c98b16 100644
--- a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
+++ b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
@@ -23,11 +23,15 @@
import java.util.Map;
import java.util.Set;
+import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
import org.apache.maven.surefire.api.provider.ProviderParameters;
import org.apache.maven.surefire.api.testset.TestListResolver;
import org.apache.maven.surefire.api.testset.TestRequest;
import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.platform.commons.JUnitException;
import org.junit.platform.engine.TestDescriptor;
import org.junit.platform.engine.TestSource;
import org.junit.platform.engine.TestTag;
@@ -40,9 +44,14 @@
import static org.apache.maven.surefire.api.booter.ProviderParameterNames.EXCLUDEDGROUPS_PROP;
import static org.apache.maven.surefire.api.booter.ProviderParameterNames.GROUPS_PROP;
import static org.apache.maven.surefire.api.booter.ProviderParameterNames.JUNIT_VINTAGE_DETECTED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
@@ -107,11 +116,103 @@ public void categoryFiltersIgnoreMethodsThatCannotBeFound() {
assertFalse(isIncluded(provider(true, "SlowTests", null), unknown));
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void groupsIncludeTaggedTestsWhenTestNgIsOnTheClasspath(boolean vintageDetected) {
+ JUnitPlatformProvider provider = provider(vintageDetected, true, "base", null);
+
+ assertTrue(isIncluded(provider, jupiterTest("baseTest", "base")));
+ assertFalse(isIncluded(provider, jupiterTest("otherTest", "other")));
+ assertFalse(isIncluded(provider, jupiterTest("untaggedTest")));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void excludedGroupsExcludeTaggedTestsWhenTestNgIsOnTheClasspath(boolean vintageDetected) {
+ JUnitPlatformProvider provider = provider(vintageDetected, true, null, "base");
+
+ assertFalse(isIncluded(provider, jupiterTest("baseTest", "base")));
+ assertTrue(isIncluded(provider, jupiterTest("otherTest", "other")));
+ assertTrue(isIncluded(provider, jupiterTest("untaggedTest")));
+ }
+
+ @Test
+ public void groupsStillMatchJUnit4CategoriesWhenTestNgIsOnTheClasspath() {
+ JUnitPlatformProvider including = provider(true, true, "SlowTests", null);
+ assertTrue(isIncluded(including, vintageTest("slow")));
+ assertFalse(isIncluded(including, vintageTest("fast")));
+
+ JUnitPlatformProvider excluding = provider(true, true, null, "SlowTests");
+ assertFalse(isIncluded(excluding, vintageTest("slow")));
+ assertTrue(isIncluded(excluding, vintageTest("fast")));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testNgTestsAreLeftToTestNg(boolean vintageDetected) {
+ TestDescriptor slowTestNgTest = testNgTest("slowTest", "slow");
+
+ JUnitPlatformProvider including = provider(vintageDetected, true, "fast", null);
+ assertTrue(isIncluded(including, slowTestNgTest));
+ assertEquals("fast", including.getConfigurationParameters().get("testng.groups"));
+
+ JUnitPlatformProvider excluding = provider(vintageDetected, true, null, "slow");
+ assertTrue(isIncluded(excluding, slowTestNgTest));
+ assertEquals("slow", excluding.getConfigurationParameters().get("testng.excludedGroups"));
+ }
+
+ @Test
+ public void groupsWhichAreNotTagExpressionsOnlyFilterTestNgTests() {
+ // fine for TestNG, but not a valid tag expression for JUnit
+ JUnitPlatformProvider provider = provider(false, true, "(?!slow).*, fast", null);
+ assertTrue(isIncluded(provider, jupiterTest("fastTest", "fast")));
+ assertFalse(isIncluded(provider, jupiterTest("slowTest", "slow")));
+ assertEquals("(?!slow).*, fast", provider.getConfigurationParameters().get("testng.groups"));
+
+ JUnitPlatformProvider onlyTestNg = provider(true, true, "(?!slow).*", "(?!fast).*");
+ assertTrue(isIncluded(onlyTestNg, jupiterTest("slowTest", "slow")));
+ assertTrue(isIncluded(onlyTestNg, jupiterTest("untaggedTest")));
+ }
+
+ @Test
+ public void groupsWhichAreNotTagExpressionsStillFailWithoutTestNg() {
+ assertThrows(JUnitException.class, () -> provider(false, false, "(?!slow).*", null));
+ }
+
+ @Test
+ public void warnsAboutGroupsWhichOnlyTestNgUnderstands() {
+ ConsoleLogger logger = mock(ConsoleLogger.class);
+
+ provider(false, true, "(?!slow).*, fast", null).warnAboutTestNgOnlyGroups(logger);
+
+ verify(logger)
+ .warning(
+ "These groups are not valid JUnit tag expressions, so they only filter TestNG tests: (?!slow).*");
+ }
+
+ @Test
+ public void doesNotWarnAboutGroupsWhichAreTagExpressions() {
+ ConsoleLogger logger = mock(ConsoleLogger.class);
+
+ // no warning when JUnit can parse the group, even a regular expression
+ provider(false, true, "fast, foo\\..*", null).warnAboutTestNgOnlyGroups(logger);
+
+ verify(logger, never()).warning(any());
+ }
+
private static JUnitPlatformProvider provider(boolean vintageDetected, String groups, String excludedGroups) {
+ return provider(vintageDetected, false, groups, excludedGroups);
+ }
+
+ private static JUnitPlatformProvider provider(
+ boolean vintageDetected, boolean testNgDetected, String groups, String excludedGroups) {
Map<String, String> properties = new HashMap<>();
if (vintageDetected) {
properties.put(JUNIT_VINTAGE_DETECTED, "true");
}
+ if (testNgDetected) {
+ properties.put("testng.version", "6.14.3");
+ }
if (groups != null) {
properties.put(GROUPS_PROP, groups);
}
@@ -140,6 +241,13 @@ private static TestDescriptor jupiterTest(String name, String... tags) {
tags);
}
+ private static TestDescriptor testNgTest(String name, String... groups) {
+ return testDescriptor(
+ UniqueId.forEngine("testng").append("method", name),
+ MethodSource.from(CategorizedTestClass.class.getName(), "fast"),
+ groups);
+ }
+
private static TestDescriptor vintageTest(String methodName) {
return testDescriptor(
UniqueId.forEngine("junit-vintage").append("test", methodName),