You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ti...@apache.org on 2019/10/02 15:37:35 UTC
[maven-surefire] 01/01: [SUREFIRE-1584] Add support for
rerunFailingTests to JUnit5 Provider
This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch SUREFIRE-1584
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
commit cfe702a796be6508932513c39dd431a8af11b250
Author: Col-E <mc...@gmu.edu>
AuthorDate: Thu May 23 03:34:24 2019 -0400
[SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
---
.../maven/plugin/failsafe/IntegrationTestMojo.java | 2 +-
.../maven/plugin/surefire/SurefirePlugin.java | 2 +-
.../src/site/apt/featurematrix.apt.vm | 4 +-
.../its/JUnitPlatformRerunFailingTestsIT.java | 281 +++++++++++++++++++++
.../junit-platform-rerun-failing-tests/pom.xml | 55 ++++
.../java/junitplatform/FlakyFirstTimeTest.java | 63 +++++
.../src/test/java/junitplatform/PassingTest.java | 38 +++
.../junitplatform/JUnitPlatformProvider.java | 40 ++-
.../surefire/junitplatform/RunListenerAdapter.java | 57 ++++-
9 files changed, 527 insertions(+), 15 deletions(-)
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index d900e37..35b0d41 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -264,7 +264,7 @@ public class IntegrationTestMojo
private String encoding;
/**
- * (JUnit 4+ providers)
+ * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
* The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
* they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
* However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index 37bf3cf..aad9230 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -245,7 +245,7 @@ public class SurefirePlugin
private String encoding;
/**
- * (JUnit 4+ providers)
+ * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
* The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
* they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
* However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
index 521895a..05d13bc 100644
--- a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
+++ b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
@@ -42,7 +42,7 @@ Feature Matrix
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
| custom run-listener | N | Y | Y | Y | - | N |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
-| re-run count | N | Y | Y | N | N | N |
+| re-run count | N | Y | Y | N | N | Y(*3) |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
| skip after failure count | N | Y | Y | Y | N | N |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
@@ -62,3 +62,5 @@ Feature Matrix
(*2) 3 extensions related to JUnit5 annotation <<<DisplayName>>>.
+ (*3) Since 3.0.0-M4
+
\ No newline at end of file
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
new file mode 100644
index 0000000..6d881d3
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
@@ -0,0 +1,281 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ * @author Matt Coley
+ */
+public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
+{
+ private final static String VERSION = "5.5.1";
+
+ private SurefireLauncher unpack()
+ {
+ return unpack( "/junit-platform-rerun-failing-tests" );
+ }
+
+ @Before
+ public void setup()
+ {
+ assumeJavaVersion( 1.8 );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsWithOneRetry() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsTwoRetry() throws Exception
+ {
+ // Four flakes, both tests have been re-run twice
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging()
+ .addGoal( "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
+ 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsFalse() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion(
+ VERSION ).maven().withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-DforkCount=3" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestClass() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestMethod() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+ }
+
+ private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+ }
+
+ private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+ {
+ verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+ }
+
+ private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ outputValidator.verifyTextInLog( "Errors:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Flakes:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "ERROR" );
+ outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ if ( flakes > 0 )
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: " + flakes );
+ }
+ else
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+ }
+ }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
new file mode 100644
index 0000000..af4dde0
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>junit-platform-rerun-failing-tests</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for rerun failing tests in JUnit 5/Platform</name>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
new file mode 100644
index 0000000..5c53d6d
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
@@ -0,0 +1,63 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+public class FlakyFirstTimeTest
+{
+ private static int failingCount = 0;
+
+ private static int errorCount = 0;
+
+
+ @Test
+ public void testFailingTestOne()
+ {
+ System.out.println( "Failing test" );
+ // This test will fail with only one retry, but will pass with two
+ if ( failingCount < 2 )
+ {
+ failingCount++;
+ fail( "Failing test" );
+ }
+ }
+
+ @Test
+ public void testErrorTestOne() throws Exception
+ {
+ System.out.println( "Error test" );
+ // This test will error out with only one retry, but will pass with two
+ if ( errorCount < 2 )
+ {
+ errorCount++;
+ throw new IllegalArgumentException( "..." );
+ }
+ }
+
+ @Test
+ public void testPassingTest() throws Exception
+ {
+ System.out.println( "Passing test" );
+ }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
new file mode 100644
index 0000000..4f80f9f
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
@@ -0,0 +1,38 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+
+public class PassingTest
+{
+ @Test
+ public void testPassingTestOne()
+ {
+ System.out.println( "Passing test one" );
+ }
+
+ @Test
+ public void testPassingTestTwo() throws Exception
+ {
+ System.out.println( "Passing test two" );
+ }
+}
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
index 1138e4a..4cd6342 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -31,6 +31,7 @@ import static org.apache.maven.surefire.report.ConsoleOutputCapture.startCapture
import static org.apache.maven.surefire.util.TestsToRun.fromClass;
import static org.junit.platform.commons.util.StringUtils.isBlank;
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
import java.io.IOException;
@@ -60,6 +61,7 @@ import org.junit.platform.engine.Filter;
import org.junit.platform.launcher.Launcher;
import org.junit.platform.launcher.LauncherDiscoveryRequest;
import org.junit.platform.launcher.TagFilter;
+import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
import org.junit.platform.launcher.core.LauncherFactory;
@@ -147,7 +149,26 @@ public class JUnitPlatformProvider
private void invokeAllTests( TestsToRun testsToRun, RunListener runListener )
{
LauncherDiscoveryRequest discoveryRequest = buildLauncherDiscoveryRequest( testsToRun );
- launcher.execute( discoveryRequest, new RunListenerAdapter( runListener ) );
+ RunListenerAdapter adapter = new RunListenerAdapter( runListener );
+ launcher.execute( discoveryRequest, adapter );
+ // Rerun failing tests if requested
+ int count = parameters.getTestRequest().getRerunFailingTestsCount();
+ if ( count > 0 && adapter.hasFailingTests() )
+ {
+ for ( int i = 0; i < count; i++ )
+ {
+ // Replace the "discoveryRequest" so that it only specifies the failing tests
+ discoveryRequest = buildLauncherDiscoveryRequestForRerunFailures( adapter );
+ // Reset adapter's recorded failures and invoke the failed tests again
+ adapter.getFailures().clear();
+ launcher.execute( discoveryRequest, adapter );
+ // If no tests fail in the rerun, we're done
+ if ( !adapter.hasFailingTests() )
+ {
+ break;
+ }
+ }
+ }
}
private LauncherDiscoveryRequest buildLauncherDiscoveryRequest( TestsToRun testsToRun )
@@ -161,6 +182,23 @@ public class JUnitPlatformProvider
return builder.build();
}
+ private LauncherDiscoveryRequest buildLauncherDiscoveryRequestForRerunFailures( RunListenerAdapter adapter )
+ {
+ LauncherDiscoveryRequestBuilder builder = request().filters( filters ).configurationParameters(
+ configurationParameters );
+ // Iterate over recorded failures
+ for ( TestIdentifier identifier : adapter.getFailures().keySet() )
+ {
+ // Extract quantified test name data
+ String[] classMethodName = adapter.toClassMethodNameWithoutPlan( identifier );
+ String className = classMethodName[1];
+ String methodName = classMethodName[3];
+ // Add filter for the specific failing method
+ builder.selectors( selectMethod( className, methodName ) );
+ }
+ return builder.build();
+ }
+
private Filter<?>[] newFilters()
{
List<Filter<?>> filters = new ArrayList<>();
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
index 29f3eeb..5004f26 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
@@ -47,6 +47,7 @@ final class RunListenerAdapter
implements TestExecutionListener
{
private final ConcurrentMap<TestIdentifier, Long> testStartTime = new ConcurrentHashMap<>();
+ private final ConcurrentMap<TestIdentifier, TestExecutionResult> failures = new ConcurrentHashMap<>();
private final RunListener runListener;
private volatile TestPlan testPlan;
@@ -59,6 +60,7 @@ final class RunListenerAdapter
public void testPlanExecutionStarted( TestPlan testPlan )
{
this.testPlan = testPlan;
+ failures.clear();
}
@Override
@@ -124,6 +126,7 @@ final class RunListenerAdapter
{
runListener.testError( createReportEntry( testIdentifier, testExecutionResult, elapsed ) );
}
+ failures.put( testIdentifier, testExecutionResult );
break;
default:
if ( isTest )
@@ -211,6 +214,16 @@ final class RunListenerAdapter
return new PojoStackTraceWriter( realClassName, realMethodName, throwable );
}
+ private String[] toClassMethodName( TestIdentifier testIdentifier )
+ {
+ return toClassMethodName( testIdentifier, true );
+ }
+
+ public String[] toClassMethodNameWithoutPlan( TestIdentifier testIdentifier )
+ {
+ return toClassMethodName( testIdentifier, false );
+ }
+
/**
* <ul>
* <li>[0] class name - used in stacktrace parser</li>
@@ -220,9 +233,11 @@ final class RunListenerAdapter
* </ul>
*
* @param testIdentifier a class or method
+ * @param usePlan {@code true} for using the test-plan to fetch sources.
+ * {@code false} to rely on fallbacks.
* @return 4 elements string array
*/
- private String[] toClassMethodName( TestIdentifier testIdentifier )
+ private String[] toClassMethodName( TestIdentifier testIdentifier, boolean usePlan )
{
Optional<TestSource> testSource = testIdentifier.getSource();
String display = testIdentifier.getDisplayName();
@@ -232,16 +247,24 @@ final class RunListenerAdapter
MethodSource methodSource = testSource.map( MethodSource.class::cast ).get();
String realClassName = methodSource.getClassName();
- String[] source = testPlan.getParent( testIdentifier )
- .map( this::toClassMethodName )
- .map( s -> new String[] { s[0], s[1] } )
- .orElse( new String[] { realClassName, realClassName } );
+ String[] source;
+ if ( usePlan )
+ {
+ source = testPlan.getParent( testIdentifier )
+ .map( i -> toClassMethodName( i, usePlan ) )
+ .map( s -> new String[] { s[0], s[1] } )
+ .orElse( new String[] { realClassName, realClassName } );
+ }
+ else
+ {
+ source = new String[] { realClassName, realClassName };
+ }
String methodName = methodSource.getMethodName();
boolean useMethod = display.equals( methodName ) || display.equals( methodName + "()" );
String resolvedMethodName = useMethod ? methodName : display;
- return new String[] { source[0], source[1], methodName, resolvedMethodName };
+ return new String[] {source[0], source[1], methodName, resolvedMethodName};
}
else if ( testSource.filter( ClassSource.class::isInstance ).isPresent() )
{
@@ -249,14 +272,26 @@ final class RunListenerAdapter
String className = classSource.getClassName();
String simpleClassName = className.substring( 1 + className.lastIndexOf( '.' ) );
String source = display.equals( simpleClassName ) ? className : display;
- return new String[] { className, source, null, null };
+ return new String[] {className, source, null, null};
}
else
{
- String source = testPlan.getParent( testIdentifier )
- .map( TestIdentifier::getDisplayName )
- .orElse( display );
- return new String[] { source, source, display, display };
+ String source = !usePlan ? display : testPlan.getParent( testIdentifier )
+ .map( TestIdentifier::getDisplayName ).orElse( display );
+ return new String[] {source, source, display, display};
}
}
+
+ /**
+ * @return Map of tests that failed.
+ */
+ public Map<TestIdentifier, TestExecutionResult> getFailures()
+ {
+ return failures;
+ }
+
+ public boolean hasFailingTests()
+ {
+ return !getFailures().isEmpty();
+ }
}