You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ti...@apache.org on 2019/10/05 19:34:17 UTC
[maven-surefire] branch SUREFIRE-1584 updated (004f667 -> d0230dc)
This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a change to branch SUREFIRE-1584
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git.
discard 004f667 [SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
new d0230dc [SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (004f667)
\
N -- N -- N refs/heads/SUREFIRE-1584 (d0230dc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[maven-surefire] 01/01: [SUREFIRE-1584] Add support for
rerunFailingTests to JUnit5 Provider
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch SUREFIRE-1584
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
commit d0230dcc93e6cd1b8171407237489fdf291cca48
Author: Col-E <mc...@gmu.edu>
AuthorDate: Thu May 23 03:34:24 2019 -0400
[SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
---
.../maven/plugin/failsafe/IntegrationTestMojo.java | 2 +-
.../maven/plugin/surefire/SurefirePlugin.java | 2 +-
.../src/site/apt/featurematrix.apt.vm | 4 +-
.../its/JUnitPlatformRerunFailingTestsIT.java | 281 +++++++++++++++++++++
.../junit-platform-rerun-failing-tests/pom.xml | 55 ++++
.../java/junitplatform/FlakyFirstTimeTest.java | 63 +++++
.../src/test/java/junitplatform/PassingTest.java | 38 +++
.../junitplatform/JUnitPlatformProvider.java | 40 ++-
.../surefire/junitplatform/RunListenerAdapter.java | 63 ++++-
.../junitplatform/JUnitPlatformProviderTest.java | 185 ++++++++++++++
10 files changed, 718 insertions(+), 15 deletions(-)
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index d900e37..35b0d41 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -264,7 +264,7 @@ public class IntegrationTestMojo
private String encoding;
/**
- * (JUnit 4+ providers)
+ * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
* The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
* they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
* However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index 37bf3cf..aad9230 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -245,7 +245,7 @@ public class SurefirePlugin
private String encoding;
/**
- * (JUnit 4+ providers)
+ * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
* The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
* they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
* However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
index 521895a..05d13bc 100644
--- a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
+++ b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
@@ -42,7 +42,7 @@ Feature Matrix
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
| custom run-listener | N | Y | Y | Y | - | N |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
-| re-run count | N | Y | Y | N | N | N |
+| re-run count | N | Y | Y | N | N | Y(*3) |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
| skip after failure count | N | Y | Y | Y | N | N |
*---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
@@ -62,3 +62,5 @@ Feature Matrix
(*2) 3 extensions related to JUnit5 annotation <<<DisplayName>>>.
+ (*3) Since 3.0.0-M4
+
\ No newline at end of file
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
new file mode 100644
index 0000000..6d881d3
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
@@ -0,0 +1,281 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ * @author Matt Coley
+ */
+public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
+{
+ private final static String VERSION = "5.5.1";
+
+ private SurefireLauncher unpack()
+ {
+ return unpack( "/junit-platform-rerun-failing-tests" );
+ }
+
+ @Before
+ public void setup()
+ {
+ assumeJavaVersion( 1.8 );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsWithOneRetry() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsTwoRetry() throws Exception
+ {
+ // Four flakes, both tests have been re-run twice
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging()
+ .addGoal( "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
+ 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsFalse() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion(
+ VERSION ).maven().withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-DforkCount=3" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestClass() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestMethod() throws Exception
+ {
+ OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+ .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+ }
+
+ private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+ }
+
+ private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+ {
+ verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+ }
+
+ private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ outputValidator.verifyTextInLog( "Errors:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Flakes:" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failures:" );
+ outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "ERROR" );
+ outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ if ( flakes > 0 )
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: " + flakes );
+ }
+ else
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+ }
+ }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
new file mode 100644
index 0000000..af4dde0
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>junit-platform-rerun-failing-tests</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for rerun failing tests in JUnit 5/Platform</name>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
new file mode 100644
index 0000000..5c53d6d
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
@@ -0,0 +1,63 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+public class FlakyFirstTimeTest
+{
+ private static int failingCount = 0;
+
+ private static int errorCount = 0;
+
+
+ @Test
+ public void testFailingTestOne()
+ {
+ System.out.println( "Failing test" );
+ // This test will fail with only one retry, but will pass with two
+ if ( failingCount < 2 )
+ {
+ failingCount++;
+ fail( "Failing test" );
+ }
+ }
+
+ @Test
+ public void testErrorTestOne() throws Exception
+ {
+ System.out.println( "Error test" );
+ // This test will error out with only one retry, but will pass with two
+ if ( errorCount < 2 )
+ {
+ errorCount++;
+ throw new IllegalArgumentException( "..." );
+ }
+ }
+
+ @Test
+ public void testPassingTest() throws Exception
+ {
+ System.out.println( "Passing test" );
+ }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
new file mode 100644
index 0000000..4f80f9f
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
@@ -0,0 +1,38 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+
+public class PassingTest
+{
+ @Test
+ public void testPassingTestOne()
+ {
+ System.out.println( "Passing test one" );
+ }
+
+ @Test
+ public void testPassingTestTwo() throws Exception
+ {
+ System.out.println( "Passing test two" );
+ }
+}
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
index 1138e4a..6c6c8e9 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -31,6 +31,7 @@ import static org.apache.maven.surefire.report.ConsoleOutputCapture.startCapture
import static org.apache.maven.surefire.util.TestsToRun.fromClass;
import static org.junit.platform.commons.util.StringUtils.isBlank;
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
import java.io.IOException;
@@ -60,6 +61,7 @@ import org.junit.platform.engine.Filter;
import org.junit.platform.launcher.Launcher;
import org.junit.platform.launcher.LauncherDiscoveryRequest;
import org.junit.platform.launcher.TagFilter;
+import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
import org.junit.platform.launcher.core.LauncherFactory;
@@ -147,7 +149,26 @@ public class JUnitPlatformProvider
private void invokeAllTests( TestsToRun testsToRun, RunListener runListener )
{
LauncherDiscoveryRequest discoveryRequest = buildLauncherDiscoveryRequest( testsToRun );
- launcher.execute( discoveryRequest, new RunListenerAdapter( runListener ) );
+ RunListenerAdapter adapter = new RunListenerAdapter( runListener );
+ launcher.execute( discoveryRequest, adapter );
+ // Rerun failing tests if requested
+ int count = parameters.getTestRequest().getRerunFailingTestsCount();
+ if ( count > 0 && adapter.hasFailingTests() )
+ {
+ for ( int i = 0; i < count; i++ )
+ {
+ // Replace the "discoveryRequest" so that it only specifies the failing tests
+ discoveryRequest = buildLauncherDiscoveryRequestForRerunFailures( adapter );
+ // Reset adapter's recorded failures and invoke the failed tests again
+ adapter.reset();
+ launcher.execute( discoveryRequest, adapter );
+ // If no tests fail in the rerun, we're done
+ if ( !adapter.hasFailingTests() )
+ {
+ break;
+ }
+ }
+ }
}
private LauncherDiscoveryRequest buildLauncherDiscoveryRequest( TestsToRun testsToRun )
@@ -161,6 +182,23 @@ public class JUnitPlatformProvider
return builder.build();
}
+ private LauncherDiscoveryRequest buildLauncherDiscoveryRequestForRerunFailures( RunListenerAdapter adapter )
+ {
+ LauncherDiscoveryRequestBuilder builder = request().filters( filters ).configurationParameters(
+ configurationParameters );
+ // Iterate over recorded failures
+ for ( TestIdentifier identifier : adapter.getFailures().keySet() )
+ {
+ // Extract quantified test name data
+ String[] classMethodName = adapter.toClassMethodNameWithoutPlan( identifier );
+ String className = classMethodName[1];
+ String methodName = classMethodName[3];
+ // Add filter for the specific failing method
+ builder.selectors( selectMethod( className, methodName ) );
+ }
+ return builder.build();
+ }
+
private Filter<?>[] newFilters()
{
List<Filter<?>> filters = new ArrayList<>();
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
index 29f3eeb..f16e2a0 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
@@ -47,6 +47,7 @@ final class RunListenerAdapter
implements TestExecutionListener
{
private final ConcurrentMap<TestIdentifier, Long> testStartTime = new ConcurrentHashMap<>();
+ private final ConcurrentMap<TestIdentifier, TestExecutionResult> failures = new ConcurrentHashMap<>();
private final RunListener runListener;
private volatile TestPlan testPlan;
@@ -59,6 +60,7 @@ final class RunListenerAdapter
public void testPlanExecutionStarted( TestPlan testPlan )
{
this.testPlan = testPlan;
+ failures.clear();
}
@Override
@@ -124,6 +126,7 @@ final class RunListenerAdapter
{
runListener.testError( createReportEntry( testIdentifier, testExecutionResult, elapsed ) );
}
+ failures.put( testIdentifier, testExecutionResult );
break;
default:
if ( isTest )
@@ -211,6 +214,16 @@ final class RunListenerAdapter
return new PojoStackTraceWriter( realClassName, realMethodName, throwable );
}
+ private String[] toClassMethodName( TestIdentifier testIdentifier )
+ {
+ return toClassMethodName( testIdentifier, true );
+ }
+
+ String[] toClassMethodNameWithoutPlan( TestIdentifier testIdentifier )
+ {
+ return toClassMethodName( testIdentifier, false );
+ }
+
/**
* <ul>
* <li>[0] class name - used in stacktrace parser</li>
@@ -220,9 +233,11 @@ final class RunListenerAdapter
* </ul>
*
* @param testIdentifier a class or method
+ * @param usePlan {@code true} for using the test-plan to fetch sources.
+ * {@code false} to rely on fallbacks.
* @return 4 elements string array
*/
- private String[] toClassMethodName( TestIdentifier testIdentifier )
+ private String[] toClassMethodName( TestIdentifier testIdentifier, boolean usePlan )
{
Optional<TestSource> testSource = testIdentifier.getSource();
String display = testIdentifier.getDisplayName();
@@ -232,16 +247,24 @@ final class RunListenerAdapter
MethodSource methodSource = testSource.map( MethodSource.class::cast ).get();
String realClassName = methodSource.getClassName();
- String[] source = testPlan.getParent( testIdentifier )
- .map( this::toClassMethodName )
- .map( s -> new String[] { s[0], s[1] } )
- .orElse( new String[] { realClassName, realClassName } );
+ String[] source;
+ if ( usePlan )
+ {
+ source = testPlan.getParent( testIdentifier )
+ .map( i -> toClassMethodName( i, usePlan ) )
+ .map( s -> new String[] { s[0], s[1] } )
+ .orElse( new String[] { realClassName, realClassName } );
+ }
+ else
+ {
+ source = new String[] { realClassName, realClassName };
+ }
String methodName = methodSource.getMethodName();
boolean useMethod = display.equals( methodName ) || display.equals( methodName + "()" );
String resolvedMethodName = useMethod ? methodName : display;
- return new String[] { source[0], source[1], methodName, resolvedMethodName };
+ return new String[] {source[0], source[1], methodName, resolvedMethodName};
}
else if ( testSource.filter( ClassSource.class::isInstance ).isPresent() )
{
@@ -249,14 +272,32 @@ final class RunListenerAdapter
String className = classSource.getClassName();
String simpleClassName = className.substring( 1 + className.lastIndexOf( '.' ) );
String source = display.equals( simpleClassName ) ? className : display;
- return new String[] { className, source, null, null };
+ return new String[] {className, source, null, null};
}
else
{
- String source = testPlan.getParent( testIdentifier )
- .map( TestIdentifier::getDisplayName )
- .orElse( display );
- return new String[] { source, source, display, display };
+ String source = !usePlan ? display : testPlan.getParent( testIdentifier )
+ .map( TestIdentifier::getDisplayName ).orElse( display );
+ return new String[] {source, source, display, display};
}
}
+
+ /**
+ * @return Map of tests that failed.
+ */
+ Map<TestIdentifier, TestExecutionResult> getFailures()
+ {
+ return failures;
+ }
+
+ boolean hasFailingTests()
+ {
+ return !getFailures().isEmpty();
+ }
+
+ void reset()
+ {
+ getFailures().clear();
+ testPlan = null;
+ }
}
diff --git a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java
index 3e82a15..feb169c 100644
--- a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java
+++ b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java
@@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@@ -148,6 +149,127 @@ public class JUnitPlatformProviderTest
}
@Test
+ public void failingTestCaseAfterRerun()
+ throws Exception
+ {
+ Launcher launcher = LauncherFactory.create();
+ ProviderParameters parameters = providerParametersMock();
+ // Mock the rerun variable
+ when( parameters.getTestRequest().getRerunFailingTestsCount() ).thenReturn( 1 );
+
+ JUnitPlatformProvider provider = new JUnitPlatformProvider( parameters, launcher );
+
+ TestPlanSummaryListener executionListener = new TestPlanSummaryListener();
+ launcher.registerTestExecutionListeners( executionListener );
+
+ invokeProvider( provider, TestClass2.class );
+
+ assertThat( executionListener.summaries ).hasSize( 2 );
+ TestExecutionSummary summary = executionListener.summaries.get( 0 );
+ assertEquals( TestClass2.TESTS_FOUND, summary.getTestsFoundCount() );
+ assertEquals( TestClass2.TESTS_STARTED, summary.getTestsStartedCount() );
+ assertEquals( TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount() );
+ assertEquals( TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount() );
+ assertEquals( TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount() );
+ assertEquals( TestClass2.TESTS_FAILED, summary.getTestsFailedCount() );
+
+ // Should only be re-running one unit test
+ // - And it should only fail
+ summary = executionListener.summaries.get( 1 );
+ assertEquals( 1, summary.getTestsFoundCount() );
+ assertEquals( 1, summary.getTestsStartedCount() );
+ assertEquals( 0, summary.getTestsSkippedCount() );
+ assertEquals( 0, summary.getTestsSucceededCount() );
+ assertEquals( 0, summary.getTestsAbortedCount() );
+ assertEquals( 1, summary.getTestsFailedCount() );
+ }
+
+ @Test
+ public void rerunStillFailing()
+ throws Exception
+ {
+ Launcher launcher = LauncherFactory.create();
+ ProviderParameters parameters = providerParametersMock();
+ // Mock the rerun variable
+ when( parameters.getTestRequest().getRerunFailingTestsCount() ).thenReturn( 2 );
+
+ JUnitPlatformProvider provider = new JUnitPlatformProvider( parameters, launcher );
+ TestPlanSummaryListener executionListener = new TestPlanSummaryListener();
+ launcher.registerTestExecutionListeners( executionListener );
+
+ // 3 unit tests:
+ // - fail always
+ // - fail twice (pass on 2nd rerun)
+ // - pass always
+ invokeProvider( provider, TestClass4.class );
+
+ assertThat( executionListener.summaries ).hasSize( 3 );
+ TestExecutionSummary summary = executionListener.summaries.get( 0 );
+ assertEquals( 6, summary.getTestsFoundCount() );
+ assertEquals( 5, summary.getTestsStartedCount() );
+ assertEquals( 1, summary.getTestsSkippedCount() );
+ assertEquals( 1, summary.getTestsSucceededCount() );
+ assertEquals( 1, summary.getTestsAbortedCount() );
+ assertEquals( 3, summary.getTestsFailedCount() );
+
+ // Should rerun both of the failures
+ summary = executionListener.summaries.get( 1 );
+ assertEquals( 3, summary.getTestsFoundCount() );
+ assertEquals( 3, summary.getTestsStartedCount() );
+ assertEquals( 0, summary.getTestsSkippedCount() );
+ assertEquals( 0, summary.getTestsSucceededCount() );
+ assertEquals( 0, summary.getTestsAbortedCount() );
+ assertEquals( 3, summary.getTestsFailedCount() );
+
+ // now only one failure should remain
+ summary = executionListener.summaries.get( 2 );
+ assertEquals( 3, summary.getTestsFoundCount() );
+ assertEquals( 3, summary.getTestsStartedCount() );
+ assertEquals( 0, summary.getTestsSkippedCount() );
+ assertEquals( 1, summary.getTestsSucceededCount() );
+ assertEquals( 0, summary.getTestsAbortedCount() );
+ assertEquals( 2, summary.getTestsFailedCount() );
+ }
+
+ @Test
+ public void rerunWithSuccess()
+ throws Exception
+ {
+ Launcher launcher = LauncherFactory.create();
+ ProviderParameters parameters = providerParametersMock();
+ // Mock the rerun variable
+ when( parameters.getTestRequest().getRerunFailingTestsCount() ).thenReturn( 2 );
+ when( parameters.getProviderProperties() )
+ .thenReturn( singletonMap( JUnitPlatformProvider.CONFIGURATION_PARAMETERS,
+ "forkCount = 1\nreuseForks = true" ) );
+
+ JUnitPlatformProvider provider = new JUnitPlatformProvider( parameters, launcher );
+ TestPlanSummaryListener executionListener = new TestPlanSummaryListener();
+ launcher.registerTestExecutionListeners( executionListener );
+
+ invokeProvider( provider, TestClass5.class );
+
+ assertThat( executionListener.summaries ).hasSize( 3 );
+ TestExecutionSummary summary = executionListener.summaries.get( 0 );
+ assertEquals( 1, summary.getTestsFoundCount() );
+ assertEquals( 1, summary.getTestsStartedCount() );
+ assertEquals( 0, summary.getTestsSucceededCount() );
+ assertEquals( 1, summary.getTestsFailedCount() );
+
+ summary = executionListener.summaries.get( 1 );
+ assertEquals( 1, summary.getTestsFoundCount() );
+ assertEquals( 1, summary.getTestsStartedCount() );
+ assertEquals( 0, summary.getTestsSucceededCount() );
+ assertEquals( 1, summary.getTestsFailedCount() );
+
+ summary = executionListener.summaries.get( 2 );
+ assertEquals( 1, summary.getTestsFoundCount() );
+ assertEquals( 1, summary.getTestsStartedCount() );
+ assertEquals( 1, summary.getTestsSucceededCount() );
+ assertEquals( 0, summary.getTestsFailedCount() );
+ }
+
+ @Test
public void allDiscoveredTestsAreInvokedForNullArgument()
throws Exception
{
@@ -707,4 +829,67 @@ public class JUnitPlatformProviderTest
throw new RuntimeException();
}
}
+
+ static class TestClass4
+ {
+ static int count;
+
+ @org.junit.jupiter.api.Test
+ void testPass()
+ {
+ }
+
+ @org.junit.jupiter.api.Test
+ void testAborted()
+ {
+ assumeFalse( true );
+ throw new IllegalStateException( "this exception should neve happen" );
+ }
+
+ @org.junit.jupiter.api.Test
+ void testAlwaysError()
+ {
+ throw new Error( "some error" );
+ }
+
+ @org.junit.jupiter.api.Test
+ void testAlwaysFail()
+ {
+ assertTrue( false );
+ }
+
+ @org.junit.jupiter.api.Test
+ @org.junit.jupiter.api.Disabled
+ void testAlwaysSkipped()
+ {
+ throw new IllegalStateException( "this test should be never called" );
+ }
+
+ @org.junit.jupiter.api.Test
+ void testFailTwice()
+ {
+ count += 1;
+ assertTrue( count >= 3 );
+ }
+ }
+
+ static class TestClass5
+ {
+ static int count;
+
+ @org.junit.jupiter.api.Test
+ void testFailTwice1()
+ {
+ count += 1;
+ assertTrue( count >= 3 );
+ }
+
+ /*@org.junit.jupiter.api.Test
+ @org.junit.jupiter.api.Order( 2 )
+ void testFailTwice2()
+ {
+ count += 1;
+ assertTrue( count >= 3 );
+ }*/
+ }
}