You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ol...@apache.org on 2026/10/18 10:35:35 UTC

(maven-surefire) branch master updated: Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)

This is an automated email from the ASF dual-hosted git repository.

olamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git


The following commit(s) were added to refs/heads/master by this push:
     new 907a2e216 Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)
907a2e216 is described below

commit 907a2e21646744b5f4707e4b0b4a05b4358e9c83
Author: Olivier Lamy <ol...@apache.org>
AuthorDate: Sun Oct 18 20:35:28 2026 +1000

    Fix groups and excludedGroups being ignored when TestNG is on the classpath (#3534)
    
    Since 3.6.0, having TestNG on the test classpath turned off group
    filtering for JUnit 5 and JUnit 4 tests, so groups only reached TestNG.
    
    Signed-off-by: Olivier Lamy <ol...@apache.org>
---
 .../src/site/markdown/architecture.md              |  11 +-
 .../its/GroupsWithTestNgOnClasspathIT.java         | 109 ++++++++++++++++++
 .../groups-with-testng-and-jupiter-tests/pom.xml   |  87 ++++++++++++++
 .../src/test/java/mixed/FastJupiterTest.java       |  32 ++++++
 .../src/test/java/mixed/FastTestNgTest.java        |  30 +++++
 .../src/test/java/mixed/SlowJupiterTest.java       |  32 ++++++
 .../src/test/java/mixed/SlowTestNgTest.java        |  30 +++++
 .../groups-with-testng-on-classpath/pom.xml        |  79 +++++++++++++
 .../src/test/java/jupiteronly/Base.java            |  34 ++++++
 .../src/test/java/jupiteronly/BaseTest.java        |  31 +++++
 .../test/java/jupiteronly/LifecycleSupport.java    |  33 ++++++
 .../src/test/java/jupiteronly/OtherTest.java       |  30 +++++
 .../junitplatform/JUnitPlatformProvider.java       | 125 ++++++++++++++++-----
 .../JUnitPlatformProviderGroupsTest.java           | 108 ++++++++++++++++++
 14 files changed, 736 insertions(+), 35 deletions(-)

diff --git a/maven-surefire-plugin/src/site/markdown/architecture.md b/maven-surefire-plugin/src/site/markdown/architecture.md
index e1edaf09a..4a8fe9205 100644
--- a/maven-surefire-plugin/src/site/markdown/architecture.md
+++ b/maven-surefire-plugin/src/site/markdown/architecture.md
@@ -439,16 +439,17 @@ The `maven-surefire-report-plugin` reads the XML files (via `surefire-report-par
 
 ## Group / Category Filtering
 
-Since 3.6.0, test filtering by groups, categories, or tags is handled via the JUnit Platform's native **tag expression** syntax.
+Since 3.6.0, test filtering by groups, categories, or tags is handled by the JUnit Platform provider, and each kind of test is matched in its own way.
 
 ### How it works
 
 1. Users configure `<groups>` and `<excludedGroups>` in the plugin configuration
-2. For **JUnit 4**: `@Category` annotations are mapped to JUnit Platform tags via the Vintage Engine
-3. For **TestNG**: groups are mapped to JUnit Platform tags via the TestNG Engine
-4. For **JUnit 5/6**: `@Tag` annotations are used natively
+2. For **JUnit 4**: tests are matched against their `@Category` annotations, by fully qualified or simple class name
+3. For **TestNG**: the groups are handed to the TestNG Engine as `testng.groups` and `testng.excludedGroups`, and TestNG matches them itself, regular expressions included
+4. For **JUnit 5/6** and all other engines: `@Tag` annotations are matched with the JUnit Platform's native **tag expression** syntax
 
-Tag filter expressions are passed directly to the JUnit Platform Launcher.
+This does not change when TestNG is on the test classpath: the tests of the other engines are still filtered.
+A group which TestNG accepts but JUnit cannot parse as a tag expression, like `(?!slow).*`, only filters TestNG tests, and Surefire logs a warning about it.
 
 ---
 
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java
new file mode 100644
index 000000000..aa6bec2f4
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/GroupsWithTestNgOnClasspathIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Checks that groups and excludedGroups still filter Jupiter tests when TestNG is on the classpath,
+ * and that TestNG still filters its own tests.
+ */
+public class GroupsWithTestNgOnClasspathIT extends SurefireJUnit4IntegrationTestCase {
+
+    @Test
+    void groupsOnlyRunTaggedJupiterTestsWhenTestNgIsOnTheClasspath() {
+        OutputValidator outputValidator = unpack("groups-with-testng-on-classpath")
+                .setGroups("base")
+                .executeTest()
+                .verifyTextInLog("TestNG is present")
+                .verifyErrorFree(1);
+
+        assertReport(outputValidator, "jupiteronly.BaseTest", true);
+        assertReport(outputValidator, "jupiteronly.OtherTest", false);
+    }
+
+    @Test
+    void excludedGroupsSkipTaggedJupiterTestsWhenTestNgIsOnTheClasspath() {
+        OutputValidator outputValidator = unpack("groups-with-testng-on-classpath")
+                .setExcludedGroups("base")
+                .executeTest()
+                .verifyTextInLog("TestNG is present")
+                .verifyErrorFree(1);
+
+        assertReport(outputValidator, "jupiteronly.BaseTest", false);
+        assertReport(outputValidator, "jupiteronly.OtherTest", true);
+    }
+
+    @Test
+    void groupsFilterTestNgAndJupiterTests() {
+        OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+                .setGroups("fast")
+                .executeTest()
+                .verifyErrorFree(2);
+
+        assertOnlyFastTestsRan(outputValidator);
+    }
+
+    @Test
+    void excludedGroupsFilterTestNgAndJupiterTests() {
+        OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+                .setExcludedGroups("slow")
+                .executeTest()
+                .verifyErrorFree(2);
+
+        assertOnlyFastTestsRan(outputValidator);
+    }
+
+    @Test
+    void groupsWhichAreNotTagExpressionsOnlyFilterTestNgTests() {
+        // (?!slow).* works for TestNG, but JUnit can't parse it as a tag expression
+        OutputValidator outputValidator = unpack("groups-with-testng-and-jupiter-tests")
+                .activateProfile("testng-regex")
+                .executeTest()
+                .verifyTextInLog(
+                        "These groups are not valid JUnit tag expressions, so they only filter TestNG tests: (?!slow).*")
+                .verifyErrorFree(3);
+
+        assertReport(outputValidator, "mixed.FastTestNgTest", true);
+        assertReport(outputValidator, "mixed.SlowTestNgTest", false);
+        assertReport(outputValidator, "mixed.FastJupiterTest", true);
+        assertReport(outputValidator, "mixed.SlowJupiterTest", true);
+    }
+
+    private static void assertOnlyFastTestsRan(OutputValidator outputValidator) {
+        assertReport(outputValidator, "mixed.FastTestNgTest", true);
+        assertReport(outputValidator, "mixed.SlowTestNgTest", false);
+        assertReport(outputValidator, "mixed.FastJupiterTest", true);
+        assertReport(outputValidator, "mixed.SlowJupiterTest", false);
+    }
+
+    private static void assertReport(OutputValidator outputValidator, String testClass, boolean ran) {
+        if (ran) {
+            outputValidator
+                    .getSurefireReportsXmlFile("TEST-" + testClass + ".xml")
+                    .assertFileExists();
+        } else {
+            outputValidator
+                    .getSurefireReportsXmlFile("TEST-" + testClass + ".xml")
+                    .assertFileNotExists();
+        }
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml
new file mode 100644
index 000000000..3b4f38fcc
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>groups-with-testng-and-jupiter-tests</artifactId>
+  <version>1.0</version>
+  <name>Groups with TestNG and Jupiter tests side by side</name>
+
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <version>5.11.4</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Surefire adds the testng-engine on its own -->
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+      <version>7.5.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+      </plugin>
+    </plugins>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>${surefire.version}</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>testng-regex</id>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <!-- works for TestNG, but JUnit can't parse it as a tag expression -->
+              <groups>(?!slow).*</groups>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java
new file mode 100644
index 000000000..4639b9b86
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastJupiterTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("fast")
+class FastJupiterTest {
+    @Test
+    void test() {
+        assertEquals(2, 1 + 1);
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java
new file mode 100644
index 000000000..65e888bf1
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/FastTestNgTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class FastTestNgTest {
+    @Test(groups = "fast")
+    public void test() {
+        assertEquals(1 + 1, 2);
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java
new file mode 100644
index 000000000..a34e8fe2f
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowJupiterTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag("slow")
+class SlowJupiterTest {
+    @Test
+    void test() {
+        assertEquals(2, 1 + 1);
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java
new file mode 100644
index 000000000..4c4076563
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-and-jupiter-tests/src/test/java/mixed/SlowTestNgTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mixed;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class SlowTestNgTest {
+    @Test(groups = "slow")
+    public void test() {
+        assertEquals(1 + 1, 2);
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml b/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml
new file mode 100644
index 000000000..cc39db279
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/pom.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>groups-with-testng-on-classpath</artifactId>
+  <version>1.0</version>
+  <name>Jupiter tags with TestNG and JUnit 4 on the test classpath</name>
+  <description>
+    Only Jupiter tests here. JUnit 4 and TestNG are only needed by a helper class, like InjectedTest in Eclipse Sisu.
+  </description>
+
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.14.4</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- only used by the helper class, there are no JUnit 4 or TestNG tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+      <version>6.14.3</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+      </plugin>
+    </plugins>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>${surefire.version}</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+</project>
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java
new file mode 100644
index 000000000..62147da7c
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/Base.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Tag;
+
+/**
+ * Adds the base tag to a test class.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("base")
+public @interface Base {}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java
new file mode 100644
index 000000000..211bb897c
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/BaseTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Base
+class BaseTest extends LifecycleSupport {
+    @Test
+    void test() {
+        assertTrue(ready);
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java
new file mode 100644
index 000000000..c0feb47cd
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/LifecycleSupport.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+/**
+ * Helper that works with JUnit 4, JUnit 5 and TestNG, which is why all three are on the classpath.
+ */
+public abstract class LifecycleSupport {
+    protected boolean ready;
+
+    @org.junit.Before
+    @org.junit.jupiter.api.BeforeEach
+    @org.testng.annotations.BeforeMethod
+    public void setUp() {
+        ready = true;
+    }
+}
diff --git a/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java
new file mode 100644
index 000000000..d8b512051
--- /dev/null
+++ b/surefire-its/src/test/resources/groups-with-testng-on-classpath/src/test/java/jupiteronly/OtherTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package jupiteronly;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class OtherTest extends LifecycleSupport {
+    @Test
+    void test() {
+        assertTrue(ready);
+    }
+}
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
index 66481963f..6f4dc15bc 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -56,6 +56,7 @@
 import org.apache.maven.surefire.api.util.ScanResult;
 import org.apache.maven.surefire.api.util.TestsToRun;
 import org.apache.maven.surefire.shared.utils.StringUtils;
+import org.junit.platform.commons.JUnitException;
 import org.junit.platform.engine.ConfigurationParameters;
 import org.junit.platform.engine.DiscoverySelector;
 import org.junit.platform.engine.Filter;
@@ -128,6 +129,9 @@ public class JUnitPlatformProvider extends AbstractProvider {
 
     private boolean excludeTestNgEngine;
 
+    // groups only TestNG can use, we warn about them when the tests start
+    private final List<String> testNgOnlyGroups = new ArrayList<>();
+
     private final CommandChainReader commandsReader;
 
     public JUnitPlatformProvider(ProviderParameters parameters) {
@@ -225,6 +229,7 @@ public RunResult invoke(Object forkTestSet) throws TestSetFailedException, Repor
         ReporterFactory reporterFactory = parameters.getReporterFactory();
         final RunResult runResult;
         TestReportListener<TestOutputReportEntry> runListener = reporterFactory.createTestReportListener();
+        warnAboutTestNgOnlyGroups(runListener);
         CancellationTokenAdapter cancellationToken = CancellationTokenAdapter.tryCreate();
         Stoppable stoppable = prepareFailFastSupport(cancellationToken, runListener);
         RunListenerAdapter adapter = new RunListenerAdapter(runListener, stoppable);
@@ -253,6 +258,13 @@ public RunResult invoke(Object forkTestSet) throws TestSetFailedException, Repor
         return runResult;
     }
 
+    void warnAboutTestNgOnlyGroups(ConsoleLogger logger) {
+        if (!testNgOnlyGroups.isEmpty()) {
+            logger.warning("These groups are not valid JUnit tag expressions, so they only filter TestNG tests: "
+                    + String.join(", ", testNgOnlyGroups));
+        }
+    }
+
     private static void setupJunitLogger() {
         Logger logger = Logger.getLogger("org.junit");
         if (logger.getLevel() == null) {
@@ -498,33 +510,8 @@ private Filter<?>[] newFilters() {
             filters.add(includeEnclosingClasses(classNameFilter));
         }
 
-        boolean useTestNG = parameters.getProviderProperties().get("testng.version") != null;
+        filters.addAll(newGroupFilters());
 
-        if (!Boolean.parseBoolean(parameters.getProviderProperties().get(JUNIT_VINTAGE_DETECTED)) && !useTestNG) {
-            getPropertiesList(GROUPS_PROP).map(TagFilter::includeTags).ifPresent(filters::add);
-            getPropertiesList(EXCLUDEDGROUPS_PROP).map(TagFilter::excludeTags).ifPresent(filters::add);
-            Optional<Class<?>> categoryClass = getCategoryClass();
-            if (categoryClass.isPresent()) {
-                getPropertiesList(EXCLUDEDGROUPS_PROP)
-                        .map(strings -> byEngine(
-                                getExcludeCategoryFilter(strings, categoryClass),
-                                descriptor -> FilterResult.included("Not a JUnit 4 test")))
-                        .ifPresent(filters::add);
-            }
-        } else if (!useTestNG) {
-            // JUnit 4 tests use @Category, all other tests use tags (#3468, #3503)
-            Optional<Class<?>> categoryClass = getCategoryClass();
-            if (categoryClass.isPresent()) {
-                getPropertiesList(GROUPS_PROP)
-                        .map(strings -> byEngine(
-                                getIncludeCategoryFilter(strings, categoryClass), TagFilter.includeTags(strings)))
-                        .ifPresent(filters::add);
-                getPropertiesList(EXCLUDEDGROUPS_PROP)
-                        .map(strings -> byEngine(
-                                getExcludeCategoryFilter(strings, categoryClass), TagFilter.excludeTags(strings)))
-                        .ifPresent(filters::add);
-            }
-        }
         of(optionallyWildcardFilter(parameters.getTestRequest().getTestListResolver()))
                 .filter(f -> !f.isEmpty())
                 .filter(f -> !f.isWildcard())
@@ -542,6 +529,74 @@ private Filter<?>[] newFilters() {
         return filters.toArray(new Filter<?>[0]);
     }
 
+    /**
+     * Builds the filters for groups and excludedGroups. TestNG tests are left alone, TestNG gets the groups too
+     * and filters its own tests.
+     */
+    private List<PostDiscoveryFilter> newGroupFilters() {
+        Optional<PostDiscoveryFilter> includeTags =
+                getTagExpressions(GROUPS_PROP).map(TagFilter::includeTags);
+        Optional<PostDiscoveryFilter> excludeTags =
+                getTagExpressions(EXCLUDEDGROUPS_PROP).map(TagFilter::excludeTags);
+        Optional<Class<?>> categoryClass = getCategoryClass();
+        List<PostDiscoveryFilter> groupFilters = new ArrayList<>();
+        if (!Boolean.parseBoolean(parameters.getProviderProperties().get(JUNIT_VINTAGE_DETECTED))) {
+            includeTags.ifPresent(groupFilters::add);
+            excludeTags.ifPresent(groupFilters::add);
+            if (categoryClass.isPresent()) {
+                getPropertiesList(EXCLUDEDGROUPS_PROP)
+                        .map(strings -> byEngine(
+                                getExcludeCategoryFilter(strings, categoryClass),
+                                descriptor -> FilterResult.included("Not a JUnit 4 test")))
+                        .ifPresent(groupFilters::add);
+            }
+        } else if (categoryClass.isPresent()) {
+            // JUnit 4 tests use @Category, all other tests use tags (#3468, #3503)
+            PostDiscoveryFilter noTagExpression = descriptor -> FilterResult.included("No tag expression to match");
+            getPropertiesList(GROUPS_PROP)
+                    .map(strings -> byEngine(
+                            getIncludeCategoryFilter(strings, categoryClass), includeTags.orElse(noTagExpression)))
+                    .ifPresent(groupFilters::add);
+            getPropertiesList(EXCLUDEDGROUPS_PROP)
+                    .map(strings -> byEngine(
+                            getExcludeCategoryFilter(strings, categoryClass), excludeTags.orElse(noTagExpression)))
+                    .ifPresent(groupFilters::add);
+        }
+        return groupFilters.stream()
+                .map(JUnitPlatformProvider::leaveTestNgTestsAlone)
+                .collect(toList());
+    }
+
+    /**
+     * Returns the groups JUnit can parse as tag expressions. With TestNG, a group can also be a regular expression
+     * JUnit can't parse, like {@code (?!slow).*}. We leave those to TestNG instead of failing the run.
+     */
+    private Optional<List<String>> getTagExpressions(String key) {
+        Optional<List<String>> groups = getPropertiesList(key);
+        if (!groups.isPresent() || parameters.getProviderProperties().get("testng.version") == null) {
+            return groups;
+        }
+        List<String> tagExpressions = new ArrayList<>();
+        for (String group : groups.get()) {
+            if (isTagExpression(group)) {
+                tagExpressions.add(group);
+            } else {
+                testNgOnlyGroups.add(group);
+            }
+        }
+        return tagExpressions.isEmpty() ? empty() : of(tagExpressions);
+    }
+
+    private static boolean isTagExpression(String group) {
+        try {
+            TagFilter.includeTags(group);
+            return true;
+        } catch (JUnitException e) {
+            // not PreconditionViolationException, JUnit 5.4 and older don't have it
+            return false;
+        }
+    }
+
     /**
      * Gives back the names of the classes that enclose a scanned class. They are only looked up the first time
      * someone asks, because most runs never need them.
@@ -609,15 +664,25 @@ PostDiscoveryFilter getExcludeCategoryFilter(List<String> categories, Optional<C
      * Uses the first filter for JUnit 4 tests and the second one for everything else.
      */
     private static PostDiscoveryFilter byEngine(PostDiscoveryFilter vintageFilter, PostDiscoveryFilter otherFilter) {
-        return testDescriptor ->
-                isVintage(testDescriptor) ? vintageFilter.apply(testDescriptor) : otherFilter.apply(testDescriptor);
+        return testDescriptor -> isFromEngine(testDescriptor, VINTAGE_ENGINE_ID)
+                ? vintageFilter.apply(testDescriptor)
+                : otherFilter.apply(testDescriptor);
+    }
+
+    /**
+     * TestNG tests always pass here, TestNG checks their groups itself.
+     */
+    private static PostDiscoveryFilter leaveTestNgTestsAlone(PostDiscoveryFilter filter) {
+        return testDescriptor -> isFromEngine(testDescriptor, TestNgTestClassDetector.TESTNG_ENGINE_ID)
+                ? FilterResult.included("TestNG matches the groups itself")
+                : filter.apply(testDescriptor);
     }
 
-    private static boolean isVintage(TestDescriptor testDescriptor) {
+    private static boolean isFromEngine(TestDescriptor testDescriptor, String engineId) {
         return testDescriptor
                 .getUniqueId()
                 .getEngineId()
-                .filter(VINTAGE_ENGINE_ID::equals)
+                .filter(engineId::equals)
                 .isPresent();
     }
 
diff --git a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
index c70c1806d..774c98b16 100644
--- a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
+++ b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderGroupsTest.java
@@ -23,11 +23,15 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
 import org.apache.maven.surefire.api.provider.ProviderParameters;
 import org.apache.maven.surefire.api.testset.TestListResolver;
 import org.apache.maven.surefire.api.testset.TestRequest;
 import org.junit.experimental.categories.Category;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.platform.commons.JUnitException;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestSource;
 import org.junit.platform.engine.TestTag;
@@ -40,9 +44,14 @@
 import static org.apache.maven.surefire.api.booter.ProviderParameterNames.EXCLUDEDGROUPS_PROP;
 import static org.apache.maven.surefire.api.booter.ProviderParameterNames.GROUPS_PROP;
 import static org.apache.maven.surefire.api.booter.ProviderParameterNames.JUNIT_VINTAGE_DETECTED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 /**
@@ -107,11 +116,103 @@ public void categoryFiltersIgnoreMethodsThatCannotBeFound() {
         assertFalse(isIncluded(provider(true, "SlowTests", null), unknown));
     }
 
+    @ParameterizedTest
+    @ValueSource(booleans = {true, false})
+    public void groupsIncludeTaggedTestsWhenTestNgIsOnTheClasspath(boolean vintageDetected) {
+        JUnitPlatformProvider provider = provider(vintageDetected, true, "base", null);
+
+        assertTrue(isIncluded(provider, jupiterTest("baseTest", "base")));
+        assertFalse(isIncluded(provider, jupiterTest("otherTest", "other")));
+        assertFalse(isIncluded(provider, jupiterTest("untaggedTest")));
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {true, false})
+    public void excludedGroupsExcludeTaggedTestsWhenTestNgIsOnTheClasspath(boolean vintageDetected) {
+        JUnitPlatformProvider provider = provider(vintageDetected, true, null, "base");
+
+        assertFalse(isIncluded(provider, jupiterTest("baseTest", "base")));
+        assertTrue(isIncluded(provider, jupiterTest("otherTest", "other")));
+        assertTrue(isIncluded(provider, jupiterTest("untaggedTest")));
+    }
+
+    @Test
+    public void groupsStillMatchJUnit4CategoriesWhenTestNgIsOnTheClasspath() {
+        JUnitPlatformProvider including = provider(true, true, "SlowTests", null);
+        assertTrue(isIncluded(including, vintageTest("slow")));
+        assertFalse(isIncluded(including, vintageTest("fast")));
+
+        JUnitPlatformProvider excluding = provider(true, true, null, "SlowTests");
+        assertFalse(isIncluded(excluding, vintageTest("slow")));
+        assertTrue(isIncluded(excluding, vintageTest("fast")));
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {true, false})
+    public void testNgTestsAreLeftToTestNg(boolean vintageDetected) {
+        TestDescriptor slowTestNgTest = testNgTest("slowTest", "slow");
+
+        JUnitPlatformProvider including = provider(vintageDetected, true, "fast", null);
+        assertTrue(isIncluded(including, slowTestNgTest));
+        assertEquals("fast", including.getConfigurationParameters().get("testng.groups"));
+
+        JUnitPlatformProvider excluding = provider(vintageDetected, true, null, "slow");
+        assertTrue(isIncluded(excluding, slowTestNgTest));
+        assertEquals("slow", excluding.getConfigurationParameters().get("testng.excludedGroups"));
+    }
+
+    @Test
+    public void groupsWhichAreNotTagExpressionsOnlyFilterTestNgTests() {
+        // fine for TestNG, but not a valid tag expression for JUnit
+        JUnitPlatformProvider provider = provider(false, true, "(?!slow).*, fast", null);
+        assertTrue(isIncluded(provider, jupiterTest("fastTest", "fast")));
+        assertFalse(isIncluded(provider, jupiterTest("slowTest", "slow")));
+        assertEquals("(?!slow).*, fast", provider.getConfigurationParameters().get("testng.groups"));
+
+        JUnitPlatformProvider onlyTestNg = provider(true, true, "(?!slow).*", "(?!fast).*");
+        assertTrue(isIncluded(onlyTestNg, jupiterTest("slowTest", "slow")));
+        assertTrue(isIncluded(onlyTestNg, jupiterTest("untaggedTest")));
+    }
+
+    @Test
+    public void groupsWhichAreNotTagExpressionsStillFailWithoutTestNg() {
+        assertThrows(JUnitException.class, () -> provider(false, false, "(?!slow).*", null));
+    }
+
+    @Test
+    public void warnsAboutGroupsWhichOnlyTestNgUnderstands() {
+        ConsoleLogger logger = mock(ConsoleLogger.class);
+
+        provider(false, true, "(?!slow).*, fast", null).warnAboutTestNgOnlyGroups(logger);
+
+        verify(logger)
+                .warning(
+                        "These groups are not valid JUnit tag expressions, so they only filter TestNG tests: (?!slow).*");
+    }
+
+    @Test
+    public void doesNotWarnAboutGroupsWhichAreTagExpressions() {
+        ConsoleLogger logger = mock(ConsoleLogger.class);
+
+        // no warning when JUnit can parse the group, even a regular expression
+        provider(false, true, "fast, foo\\..*", null).warnAboutTestNgOnlyGroups(logger);
+
+        verify(logger, never()).warning(any());
+    }
+
     private static JUnitPlatformProvider provider(boolean vintageDetected, String groups, String excludedGroups) {
+        return provider(vintageDetected, false, groups, excludedGroups);
+    }
+
+    private static JUnitPlatformProvider provider(
+            boolean vintageDetected, boolean testNgDetected, String groups, String excludedGroups) {
         Map<String, String> properties = new HashMap<>();
         if (vintageDetected) {
             properties.put(JUNIT_VINTAGE_DETECTED, "true");
         }
+        if (testNgDetected) {
+            properties.put("testng.version", "6.14.3");
+        }
         if (groups != null) {
             properties.put(GROUPS_PROP, groups);
         }
@@ -140,6 +241,13 @@ private static TestDescriptor jupiterTest(String name, String... tags) {
                 tags);
     }
 
+    private static TestDescriptor testNgTest(String name, String... groups) {
+        return testDescriptor(
+                UniqueId.forEngine("testng").append("method", name),
+                MethodSource.from(CategorizedTestClass.class.getName(), "fast"),
+                groups);
+    }
+
     private static TestDescriptor vintageTest(String methodName) {
         return testDescriptor(
                 UniqueId.forEngine("junit-vintage").append("test", methodName),