You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ag...@apache.org on 2014/07/31 12:47:42 UTC
[2/4] Add rerunFailingTestsCount option for maven surefire to rerun
failing tests immediately after they fail.
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
deleted file mode 100644
index 5a74c45..0000000
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-package org.apache.maven.plugin.surefire.report;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import junit.framework.AssertionFailedError;
-import junit.framework.TestCase;
-
-import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter;
-import org.apache.maven.shared.utils.StringUtils;
-import org.apache.maven.shared.utils.xml.Xpp3Dom;
-import org.apache.maven.shared.utils.xml.Xpp3DomBuilder;
-import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter;
-import org.apache.maven.surefire.report.ReportEntry;
-import org.apache.maven.surefire.report.SimpleReportEntry;
-import org.apache.maven.surefire.report.StackTraceWriter;
-
-@SuppressWarnings( "ResultOfMethodCallIgnored" )
-public class StatelessXMLReporterTest
- extends TestCase
-{
-
- private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false );
-
- private ReportEntry reportEntry;
-
- private TestSetStats stats;
-
- private File expectedReportFile;
-
- protected void setUp()
- throws Exception
- {
- super.setUp();
- reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest",
- new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 );
- stats = new TestSetStats( false, true );
- }
-
- @Override protected void tearDown()
- throws Exception
- {
- super.tearDown();
-
- if ( expectedReportFile != null )
- {
- expectedReportFile.delete();
- }
- }
-
- public void testFileNameWithoutSuffix()
- {
- File reportDir = new File( "." );
- String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest";
- reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
- WrappedReportEntry testSetReportEntry =
- new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
- stats.testSucceeded( testSetReportEntry );
- reporter.testSetCompleted( testSetReportEntry, stats );
-
- expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
- assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist",
- expectedReportFile.exists() );
- }
-
-
- public void testAllFieldsSerialized()
- throws IOException
- {
- File reportDir = new File( "." );
- String testName = "aTestMethod";
- String testName2 = "bTestMethod";
- reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
- WrappedReportEntry testSetReportEntry =
- new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
- expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
-
- stats.testSucceeded( testSetReportEntry );
- StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" );
- Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" );
- String stdOutPrefix;
- String stdErrPrefix;
- if ( defaultCharsetSupportsSpecialChar() )
- {
- stdErrPrefix = "std-\u0115rr";
- stdOutPrefix = "st]]>d-o\u00DCt";
- }
- else
- {
- stdErrPrefix = "std-err";
- stdOutPrefix = "st]]>d-out";
- }
-
- byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes();
- stdOut.write( stdOutBytes, 0, stdOutBytes.length );
-
- Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" );
-
-
- byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes();
- stdErr.write( stdErrBytes, 0, stdErrBytes.length );
- WrappedReportEntry t2 =
- new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), testName2, stackTraceWriter, 13 ),
- ReportEntryType.error, 13, stdOut, stdErr );
-
- stats.testSucceeded( t2 );
- StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false );
- reporter.testSetCompleted( testSetReportEntry, stats );
-
- FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
-
- Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") );
- assertEquals( "testsuite", testSuite.getName() );
- Xpp3Dom properties = testSuite.getChild( "properties" );
- assertEquals( System.getProperties().size(), properties.getChildCount() );
- Xpp3Dom child = properties.getChild( 1 );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
-
- Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
- Xpp3Dom tca = testcase[0];
- assertEquals( testName, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5
- assertEquals( "0.012", tca.getAttribute( "time" ) );
- assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) );
-
- Xpp3Dom tcb = testcase[1];
- assertEquals( testName2, tcb.getAttribute( "name" ) );
- assertEquals( "0.013", tcb.getAttribute( "time" ) );
- assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) );
- Xpp3Dom errorNode = tcb.getChild( "error" );
- assertNotNull( errorNode );
- assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
- assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
- assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() );
-
-
- assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() );
- }
-
- private boolean defaultCharsetSupportsSpecialChar()
- {
- // some charsets are not able to deal with \u0115 on both ways of the conversion
- return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) );
- }
-
- class Inner
- {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
new file mode 100644
index 0000000..c6c1d15
--- /dev/null
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
@@ -0,0 +1,290 @@
+package org.apache.maven.plugin.surefire.report;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.AssertionFailedError;
+import junit.framework.TestCase;
+import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter;
+import org.apache.maven.shared.utils.StringUtils;
+import org.apache.maven.shared.utils.xml.Xpp3Dom;
+import org.apache.maven.shared.utils.xml.Xpp3DomBuilder;
+import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.SimpleReportEntry;
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+@SuppressWarnings( "ResultOfMethodCallIgnored" )
+public class StatelessXmlReporterTest
+ extends TestCase
+{
+
+ private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 );
+
+ private ReportEntry reportEntry;
+
+ private TestSetStats stats;
+
+ private TestSetStats rerunStats;
+
+ private File expectedReportFile;
+
+ private final static String TEST_ONE = "aTestMethod";
+ private final static String TEST_TWO = "bTestMethod";
+ private final static String TEST_THREE = "cTestMethod";
+
+ protected void setUp()
+ throws Exception
+ {
+ super.setUp();
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest",
+ new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 );
+ stats = new TestSetStats( false, true );
+ rerunStats = new TestSetStats( false, true );
+ reporter.cleanTestHistoryMap();
+ }
+
+ @Override protected void tearDown()
+ throws Exception
+ {
+ super.tearDown();
+
+ if ( expectedReportFile != null )
+ {
+ expectedReportFile.delete();
+ }
+ }
+
+ public void testFileNameWithoutSuffix()
+ {
+ File reportDir = new File( "." );
+ String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest";
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ stats.testSucceeded( testSetReportEntry );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+
+ expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
+ assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist",
+ expectedReportFile.exists() );
+ }
+
+
+ public void testAllFieldsSerialized()
+ throws IOException
+ {
+ File reportDir = new File( "." );
+
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 );
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" );
+
+ stats.testSucceeded( testSetReportEntry );
+ StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" );
+ Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" );
+ String stdOutPrefix;
+ String stdErrPrefix;
+ if ( defaultCharsetSupportsSpecialChar() )
+ {
+ stdErrPrefix = "std-\u0115rr";
+ stdOutPrefix = "st]]>d-o\u00DCt";
+ }
+ else
+ {
+ stdErrPrefix = "std-err";
+ stdOutPrefix = "st]]>d-out";
+ }
+
+ byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes();
+ stdOut.write( stdOutBytes, 0, stdOutBytes.length );
+
+ Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" );
+
+
+ byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes();
+ stdErr.write( stdErrBytes, 0, stdErrBytes.length );
+ WrappedReportEntry t2 =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriter, 13 ),
+ ReportEntryType.error, 13, stdOut, stdErr );
+
+ stats.testSucceeded( t2 );
+ StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+
+ FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
+
+ Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") );
+ assertEquals( "testsuite", testSuite.getName() );
+ Xpp3Dom properties = testSuite.getChild( "properties" );
+ assertEquals( System.getProperties().size(), properties.getChildCount() );
+ Xpp3Dom child = properties.getChild( 1 );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+
+ Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
+ Xpp3Dom tca = testcase[0];
+ assertEquals( TEST_ONE, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5
+ assertEquals( "0.012", tca.getAttribute( "time" ) );
+ assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) );
+
+ Xpp3Dom tcb = testcase[1];
+ assertEquals( TEST_TWO, tcb.getAttribute( "name" ) );
+ assertEquals( "0.013", tcb.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) );
+ Xpp3Dom errorNode = tcb.getChild( "error" );
+ assertNotNull( errorNode );
+ assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
+ assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() );
+
+
+ assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() );
+ }
+
+ public void testOutputRerunFlakyFailure()
+ throws IOException
+ {
+ File reportDir = new File( "." );
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 );
+
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" );
+
+ stats.testSucceeded( testSetReportEntry );
+ StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed",
+ "fail at foo" );
+ StackTraceWriter stackTraceWriterTwo =
+ new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" );
+
+ String firstRunOut = "first run out";
+ String firstRunErr = "first run err";
+ String secondRunOut = "second run out";
+ String secondRunErr = "second run err";
+
+ WrappedReportEntry testTwoFirstError =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterOne, 5 ),
+ ReportEntryType.error, 5, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ WrappedReportEntry testTwoSecondError =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterTwo, 13 ),
+ ReportEntryType.error, 13, createStdOutput( secondRunOut ),
+ createStdOutput( secondRunErr ) );
+
+ WrappedReportEntry testThreeFirstRun =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterOne, 13 ),
+ ReportEntryType.failure, 13, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ WrappedReportEntry testThreeSecondRun =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterTwo, 2 ),
+ ReportEntryType.success, 2, createStdOutput( secondRunOut ),
+ createStdOutput( secondRunErr ) );
+
+ stats.testSucceeded( testTwoFirstError );
+ stats.testSucceeded( testThreeFirstRun );
+ rerunStats.testSucceeded( testTwoSecondError );
+ rerunStats.testSucceeded( testThreeSecondRun );
+
+ StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 1 );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+ reporter.testSetCompleted( testSetReportEntry, rerunStats );
+
+ FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
+
+ Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8" ) );
+ assertEquals( "testsuite", testSuite.getName() );
+ // 0.019 = 0.012 + 0.005 +0.002
+ assertEquals( "0.019", testSuite.getAttribute( "time" ) );
+ Xpp3Dom properties = testSuite.getChild( "properties" );
+ assertEquals( System.getProperties().size(), properties.getChildCount() );
+ Xpp3Dom child = properties.getChild( 1 );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+
+ Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
+ Xpp3Dom testCaseOne = testcase[0];
+ assertEquals( TEST_ONE, testCaseOne.getAttribute( "name" ) );
+ assertEquals( "0.012", testCaseOne.getAttribute( "time" ) );
+ assertEquals( this.getClass().getName(), testCaseOne.getAttribute( "classname" ) );
+
+ Xpp3Dom testCaseTwo = testcase[1];
+ assertEquals( TEST_TWO, testCaseTwo.getAttribute( "name" ) );
+ // Run time for a rerun failing test is the run time of the first run
+ assertEquals( "0.005", testCaseTwo.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), testCaseTwo.getAttribute( "classname" ) );
+ Xpp3Dom errorNode = testCaseTwo.getChild( "error" );
+ Xpp3Dom rerunErrorNode = testCaseTwo.getChild( "rerunError" );
+ assertNotNull( errorNode );
+ assertNotNull( rerunErrorNode );
+
+ assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
+
+ // Check rerun error node contains all the information
+ assertEquals( firstRunOut, testCaseTwo.getChild( "system-out" ).getValue() );
+ assertEquals( firstRunErr, testCaseTwo.getChild( "system-err" ).getValue() );
+ assertEquals( secondRunOut, rerunErrorNode.getChild( "system-out" ).getValue() );
+ assertEquals( secondRunErr, rerunErrorNode.getChild( "system-err" ).getValue() );
+ assertEquals( "A fud msg two", rerunErrorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo two", rerunErrorNode.getAttribute( "type" ) );
+
+ // Check flaky failure node
+ Xpp3Dom testCaseThree = testcase[2];
+ assertEquals( TEST_THREE, testCaseThree.getAttribute( "name" ) );
+ // Run time for a flaky test is the run time of the first successful run
+ assertEquals( "0.002", testCaseThree.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), testCaseThree.getAttribute( "classname" ) );
+ Xpp3Dom flakyFailureNode = testCaseThree.getChild( "flakyFailure" );
+ assertNotNull( flakyFailureNode );
+ assertEquals( firstRunOut, flakyFailureNode.getChild( "system-out" ).getValue() );
+ assertEquals( firstRunErr, flakyFailureNode.getChild( "system-err" ).getValue() );
+ // system-out and system-err should not be present for flaky failures
+ assertNull( testCaseThree.getChild( "system-out" ) );
+ assertNull( testCaseThree.getChild( "system-err" ) );
+ }
+
+ private boolean defaultCharsetSupportsSpecialChar()
+ {
+ // some charsets are not able to deal with \u0115 on both ways of the conversion
+ return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) );
+ }
+
+ private Utf8RecodingDeferredFileOutputStream createStdOutput( String content )
+ throws IOException
+ {
+ Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds2" );
+ stdOut.write( content.getBytes(), 0, content.length() );
+ return stdOut;
+ }
+
+ class Inner
+ {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
index ca6c469..acb17cc 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
@@ -1,7 +1,5 @@
package org.apache.maven.surefire.report;
-import java.util.Collection;
-
import junit.framework.TestCase;
/*
@@ -24,85 +22,14 @@ import junit.framework.TestCase;
public class RunStatisticsTest
extends TestCase
{
- private static final String Method = "AClass#AMethod";
-
- private static final String DUMMY_ERROR_SOURCE = Method + " RuntimeException";
-
- private static final String DUMMY_FAILURE_SOURCE = "dummy failure source";
-
- private static final String DUMMY_MESSAGE = "foo";
-
- public void testAddErrorSourceWithThrowableMessage()
- {
- RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE );
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable );
- assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE );
- }
-
- public void testAddErrorSourceWithoutThrowable()
- {
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( null );
- assertRunStatisticsHasErrorSource( statistics, Method );
- }
-
- public void testAddErrorSourceWithThrowableWithoutMessage()
- {
- RuntimeException throwable = new RuntimeException();
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable );
- assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE );
- }
-
- public void testAddFailureSourceWithThrowableMessage()
+ public void testSetRunStatistics()
{
- RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE );
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable );
- assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE );
- }
-
- public void testAddFailureSourceWithoutThrowable()
- {
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( null );
- assertRunStatisticsHasFailureSource( statistics, Method );
- }
-
- public void testAddFailureSourceWithThrowableWithoutMessage()
- {
- RuntimeException throwable = new RuntimeException();
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable );
- assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE );
- }
-
- private RunStatistics createRunStatisticsAndAddErrorSourceWithThrowable( Throwable throwable )
- {
- StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable );
RunStatistics statistics = new RunStatistics();
- statistics.addErrorSource( stackTraceWriter );
-
- return statistics;
- }
-
- private RunStatistics createRunStatisticsAndAddFailureSourceWithThrowable( Throwable throwable )
- {
- StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable );
- RunStatistics statistics = new RunStatistics();
- statistics.addFailureSource( stackTraceWriter );
-
- return statistics;
- }
-
- private void assertRunStatisticsHasErrorSource( RunStatistics statistics, String expectedErrorSource )
- {
- Collection errorSources = statistics.getErrorSources();
- assertNotNull( "No error sources.", errorSources );
- assertEquals( "Wrong number of error sources.", 1, errorSources.size() );
- assertEquals( "Wrong error sources.", expectedErrorSource, errorSources.iterator().next() );
- }
-
- private void assertRunStatisticsHasFailureSource( RunStatistics statistics, String expectedFailureSource )
- {
- Collection failureSources = statistics.getFailureSources();
- assertNotNull( "No failure sources.", failureSources );
- assertEquals( "Wrong number of failure sources.", 1, failureSources.size() );
- assertEquals( "Wrong failure sources.", expectedFailureSource, failureSources.iterator().next() );
+ statistics.set( 10, 5, 2, 1, 2 );
+ assertEquals( 10, statistics.getCompletedCount() );
+ assertEquals( 5, statistics.getErrors() );
+ assertEquals( 2, statistics.getFailures() );
+ assertEquals( 1, statistics.getSkipped() );
+ assertEquals( 2, statistics.getFlakes() );
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index 131ad74..0b8d030 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -187,6 +187,18 @@ public class SurefirePlugin
@Parameter( property = "surefire.useManifestOnlyJar", defaultValue = "true" )
private boolean useManifestOnlyJar;
+ /**
+ * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
+ * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
+ * However, all the failing attempts will be recorded.
+ */
+ @Parameter( property = "surefire.rerunFailingTestsCount", defaultValue = "0" )
+ protected int rerunFailingTestsCount;
+
+ protected int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
protected void handleSummary( RunResult summary, Exception firstForkException )
throws MojoExecutionException, MojoFailureException
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
new file mode 100644
index 0000000..42cc98c
--- /dev/null
+++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
@@ -0,0 +1,139 @@
+ ------
+ Rerun failing tests
+ ------
+ Qingzhou Luo
+ ------
+ 2014-06-27
+ ------
+
+ ~~ Licensed to the Apache Software Foundation (ASF) under one
+ ~~ or more contributor license agreements. See the NOTICE file
+ ~~ distributed with this work for additional information
+ ~~ regarding copyright ownership. The ASF licenses this file
+ ~~ to you under the Apache License, Version 2.0 (the
+ ~~ "License"); you may not use this file except in compliance
+ ~~ with the License. You may obtain a copy of the License at
+ ~~
+ ~~ http://www.apache.org/licenses/LICENSE-2.0
+ ~~
+ ~~ Unless required by applicable law or agreed to in writing,
+ ~~ software distributed under the License is distributed on an
+ ~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~~ KIND, either express or implied. See the License for the
+ ~~ specific language governing permissions and limitations
+ ~~ under the License.
+
+ ~~ NOTE: For help with the syntax of this file, see:
+ ~~ http://maven.apache.org/doxia/references/apt-format.html
+
+Rerun Failing Tests
+
+#{if}(${project.artifactId}=="maven-surefire-plugin")
+ During development, you may re-run failing tests because they are flaky.
+ To use this feature through Maven surefire, set the <<<rerunFailingTestsCount>>> property to be a value larger than 0.
+ Tests will be run until they pass or the number of reruns has been exhausted.
+
+ << NOTE : This feature is supported only for JUnit 4.x. >>
+
+
++---+
+mvn -DrerunFailingTestsCount=2 test
++---+
+
+ If <<<rerunFailingTestsCount>>> is set to a value smaller than or euqal to 0, then it will be ignored.
+
+* Output flaky re-run information on the screen
+
+ When <<<rerunFailingTestsCount>>> is set to a value larger than 0 and the test fails,
+ then it will be re-run and each run information will also be output. Each run with its number and trimmed stack trace
+ will be output.
+
+ If the test passes in its first run, then the output on the screen will be identical to the case where
+ <<<rerunFailingTestsCount>>> is not used.
+
+ It the test fails in the first run, then there are two possible cases:
+
+ 1) The test passes in one of its re-runs: the last run will be marked as PASS
+
+ For example, a test passed in its second run will output on the screen:
+
++---+
+ Run 1: ...
+ Run 2: PASS
++---+
+
+ Then this test will be counted as a flaky test. The build will be successful, but in the end of the summary of all
+ tests run, the number of flaky tests will be output on the screen, for example:
+
++---+
+ Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Flakes: 1
++---+
+
+ 2) The test fails in all of the re-runs:
+
+ For example, a test failied in all the three runs, then all the runs will be output on the screen:
+
++---+
+ Run 1: ...
+ Run 2: ...
+ Run 3: ...
++---+
+
+ Then this build will be marked as failure. The type of the failure (failed test or test in error) of this test
+ depends on its first failure.
+
+* Output flaky re-run information in test report xml
+
+ When <<<rerunFailingTestsCount>>> is set to a value larger than 0, the output xml of test report will also be extended
+ to include information of each re-run.
+
+ If the test passes in its first run, then the output xml report will be identical to the case where
+ <<<rerunFailingTestsCount>>> is not used.
+
+ It the test fails in the first run, then there are also two possible cases.
+
+ 1) The test passes in one of its re-runs:
+
+ <<<flakyFailure>>> and <<<flakyError>>> elements will be used in the generated xml report to include information
+ of each failing re-runs. <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>>
+ or <<<flakyError>>> to include information of System.out and System.err output. The original <<<system-out>>>
+ and <<<system-err>>> elements will be retained on the top level under <<<testcase>>> for the last successful run.
+
+ For example:
+
++---+
+<testcase name=".." classname=".." time="0.1">
+ <flakyFailure message="" type=""> flaky failure stack trace
+ <system-out> flaky failure </system-out>
+ </flakyFailure>
+ <system-out> success </system-out>
+</testcase>
++---+
+
+ In the xml report, the running time of a flaky test will be the running time of the <<last successful run>>.
+
+ 2) The test fails in all of the re-runs:
+
+ <<<failure>>> and <<<error>>> elements will still be used in the generated xml report to include information
+ for the first failing run, the same as without using <<<rerunFailingTests>>>. <<<rerunFailure>>> and <<<rerunError>>>
+ elements will be used in the generated xml report to include information of each <<subsequent>> failing re-runs.
+ <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>> or <<<flakyError>>> to include
+ information of System.out and System.err output. The original <<<system-out>>> and <<<system-err>>> elements will be
+ retained on the top level under <<<testcase>>> for the first original failing run.
+
+ For example:
+
++---+
+<testcase name=".." classname=".." time="0.1">
+ <failure message="" type=""> first failure stack trace </failure>
+ <system-out> first failure </system-out>
+ <rerunFailure message="" type=""> rerun failure stack trace
+ <system-out> rerun failure </system-out>
+ </rerunFailure>
+</testcase>
++---+
+
+ In the xml report, the running time of a failing test with re-runs will be the running time of the
+ <<first failing run>>.
+
+#{end}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/site.xml
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/site/site.xml b/maven-surefire-plugin/src/site/site.xml
index 3724c87..d98d763 100644
--- a/maven-surefire-plugin/src/site/site.xml
+++ b/maven-surefire-plugin/src/site/site.xml
@@ -43,6 +43,7 @@
<item name="Skipping Tests" href="examples/skipping-test.html"/>
<item name="Inclusions and Exclusions of Tests" href="examples/inclusion-exclusion.html"/>
<item name="Running a Single Test" href="examples/single-test.html"/>
+ <item name="Re-run Failing Tests" href="examples/rerun-failing-tests.html"/>
<item name="Class Loading and Forking" href="examples/class-loading.html"/>
<item name="Debugging Tests" href="examples/debugging.html"/>
<item name="Using System Properties" href="examples/system-properties.html"/>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
index 54a6b5b..2e4bee4 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -152,11 +152,11 @@ public class SurefireReflector
{
return null;
}
- Class[] arguments = { List.class, File.class, String.class, String.class };
+ Class[] arguments = { List.class, File.class, String.class, String.class, int.class };
Constructor constructor = ReflectionUtils.getConstructor( this.testRequest, arguments );
- return ReflectionUtils.newInstance( constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(),
+ return ReflectionUtils.newInstance(constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(),
suiteDefinition.getTestSourceDirectory(), suiteDefinition.getRequestedTest(),
- suiteDefinition.getRequestedTestMethod() } );
+ suiteDefinition.getRequestedTestMethod(), suiteDefinition.getRerunFailingTestsCount() });
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
index 595717b..83d74f6 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
@@ -53,6 +53,8 @@ public class RunResult
private final int skipped;
+ private final int flakes;
+
private final String failure;
private final boolean timeout;
@@ -85,14 +87,26 @@ public class RunResult
this( completedCount, errors, failures, skipped, null, false );
}
+ public RunResult( int completedCount, int errors, int failures, int skipped, int flakes )
+ {
+ this( completedCount, errors, failures, skipped, flakes, null, false );
+ }
+
public RunResult( int completedCount, int errors, int failures, int skipped, String failure, boolean timeout )
{
+ this( completedCount, errors, failures, skipped, 0, failure, timeout );
+ }
+
+ public RunResult( int completedCount, int errors, int failures, int skipped, int flakes, String failure,
+ boolean timeout )
+ {
this.completedCount = completedCount;
this.errors = errors;
this.failures = failures;
this.skipped = skipped;
this.failure = failure;
this.timeout = timeout;
+ this.flakes = flakes;
}
private static String getStackTrace( Exception e )
@@ -117,6 +131,11 @@ public class RunResult
return errors;
}
+ public int getFlakes()
+ {
+ return flakes;
+ }
+
public int getFailures()
{
return failures;
@@ -176,7 +195,8 @@ public class RunResult
int fail = getFailures() + other.getFailures();
int ign = getSkipped() + other.getSkipped();
int err = getErrors() + other.getErrors();
- return new RunResult( completed, err, fail, ign, failureMessage, timeout );
+ int flakes = getFlakes() + other.getFlakes();
+ return new RunResult( completed, err, fail, ign, flakes, failureMessage, timeout );
}
public static RunResult noTestsRun()
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
index 190b71b..a237bbe 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
@@ -36,6 +36,8 @@ public class TestRequest
private final String requestedTest;
+ private final int rerunFailingTestsCount;
+
/**
* @since 2.7.3
*/
@@ -51,10 +53,17 @@ public class TestRequest
*/
public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod )
{
+ this( createFiles( suiteXmlFiles ), testSourceDirectory, requestedTest, requestedTestMethod, 0 );
+ }
+
+ public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod,
+ int rerunFailingTestsCount )
+ {
this.suiteXmlFiles = createFiles( suiteXmlFiles );
this.testSourceDirectory = testSourceDirectory;
this.requestedTest = requestedTest;
this.requestedTestMethod = requestedTestMethod;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
/**
@@ -98,6 +107,16 @@ public class TestRequest
return requestedTestMethod;
}
+ /**
+ * How many times to rerun failing tests, issued with -Dsurefire.rerunFailingTestsCount from the command line.
+ *
+ * @return The int parameter to indicate how many times to rerun failing tests
+ */
+ public int getRerunFailingTestsCount()
+ {
+ return this.rerunFailingTestsCount;
+ }
+
private static List<File> createFiles( List suiteXmlFiles )
{
if ( suiteXmlFiles != null )
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
index 607c332..2c42c19 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
@@ -138,4 +138,22 @@ public class TestsToRun implements Iterable<Class>
}
return result.toArray( new Class[result.size()] );
}
+
+ /**
+ * Get test class which matches className
+ *
+ * @param className string used to find the test class
+ * @return Class object with the matching name, null if could not find a class with the matching name
+ */
+ public Class getClassByName( String className )
+ {
+ for ( Class clazz : this )
+ {
+ if ( clazz.getName().equals( className ) )
+ {
+ return clazz;
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
index ec99c64..007a26c 100644
--- a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
+++ b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
@@ -58,6 +58,7 @@ public class RunResultTest
assertEquals( 3, simple.getErrors() );
assertEquals( 7, simple.getFailures() );
assertEquals( 4, simple.getSkipped() );
+ assertEquals( 2, simple.getFlakes() );
}
@@ -116,8 +117,8 @@ public class RunResultTest
private RunResult getSimpleAggregate()
{
- RunResult resultOne = new RunResult( 10, 1, 3, 2 );
- RunResult resultTwo = new RunResult( 10, 2, 4, 2 );
+ RunResult resultOne = new RunResult( 10, 1, 3, 2, 1 );
+ RunResult resultTwo = new RunResult( 10, 2, 4, 2, 1 );
return resultOne.aggregate( resultTwo );
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
index c7029c0..d569c00 100644
--- a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
+++ b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
@@ -65,6 +65,14 @@ public class TestsToRunTest
assertEquals( 2, locatedClasses.length );
}
+ public void testGetClassByName()
+ {
+ TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) );
+ assertEquals( T1.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T1" ) );
+ assertEquals( T2.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T2" ) );
+ assertEquals( null, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T3" ) );
+ }
+
class T1
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
index d53bfda..2f637f0 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -47,4 +47,5 @@ public interface BooterConstants
String PROVIDER_CONFIGURATION = "providerConfiguration";
String FORKTESTSET = "forkTestSet";
String FORKTESTSET_PREFER_TESTS_FROM_IN_STREAM = "preferTestsFromInStream";
+ String RERUN_FAILING_TESTS_COUNT = "rerunFailingTestsCount";
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
index a9028e0..fce7c42 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -78,6 +78,8 @@ public class BooterDeserializer
final String runOrder = properties.getProperty( RUN_ORDER );
final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
+ final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
+
DirectoryScannerParameters dirScannerParams =
new DirectoryScannerParameters( testClassesDirectory, includesList, excludesList, specificTestsList,
properties.getBooleanObjectProperty( FAILIFNOTESTS ), runOrder );
@@ -86,7 +88,8 @@ public class BooterDeserializer
TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
TestRequest testSuiteDefinition =
- new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod );
+ new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod,
+ rerunFailingTestsCount );
ReporterConfiguration reporterConfiguration =
new ReporterConfiguration( reportsDirectory, properties.getBooleanObjectProperty( ISTRIMSTACKTRACE ) );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
index 8a40c63..7a09352 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
@@ -76,6 +76,11 @@ public class PropertiesWrapper
return Boolean.valueOf( properties.getProperty( propertyName ) );
}
+ public int getIntProperty( String propertyName )
+ {
+ return Integer.parseInt( properties.getProperty( propertyName ) );
+ }
+
public File getFileProperty( String key )
{
final String property = getProperty( key );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
new file mode 100644
index 0000000..47fb71a
--- /dev/null
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
@@ -0,0 +1,278 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Test;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ */
+public class JUnit4RerunFailingTestsIT
+ extends SurefireJUnit4IntegrationTestCase
+{
+ private SurefireLauncher unpack()
+ {
+ return unpack( "/junit4-rerun-failing-tests" );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsWithOneRetry()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0,
+ 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsTwoRetry()
+ throws Exception
+ {
+ // Four flakes, both tests have been re-run twice
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest()
+ .assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsFalse()
+ throws Exception
+ {
+ OutputValidator outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion(
+ "4.7" ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestClass()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestMethod()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+ 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+ }
+
+ private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+ }
+
+ private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+ {
+ verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+ }
+
+ private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ outputValidator.verifyTextInLog( "Tests in error" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Flaked tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "testFailingTestOne(junit4.FlakyFirstTimeTest)" );
+ outputValidator.verifyTextInLog( "ERROR" );
+ outputValidator.verifyTextInLog( "testErrorTestOne(junit4.FlakyFirstTimeTest)" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ if ( flakes > 0 )
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: "
+ + flakes );
+ }
+ else
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
index 7c8740a..c7afb92 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
@@ -40,6 +40,12 @@ public class HelperAssertions
assertTestSuiteResults( total, errors, failures, skipped, suite );
}
+ public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, File testDir )
+ {
+ IntegrationTestSuiteResults suite = parseTestResults( new File[]{ testDir } );
+ assertTestSuiteResults( total, errors, failures, skipped, flakes, suite );
+ }
+
/**
* assert that the reports in the specified testDir have the right summary statistics
*/
@@ -59,6 +65,13 @@ public class HelperAssertions
Assert.assertEquals( "wrong number of skipped", skipped, actualSuite.getSkipped() );
}
+ public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes,
+ IntegrationTestSuiteResults actualSuite )
+ {
+ assertTestSuiteResults(total, errors, failures, skipped, actualSuite);
+ Assert.assertEquals( "wrong number of flaky tests", flakes, actualSuite.getFlakes() );
+ }
+
public static IntegrationTestSuiteResults parseTestResults( File[] testDirs )
{
List<ReportTestSuite> reports = extractReports( testDirs );
@@ -77,15 +90,16 @@ public class HelperAssertions
public static IntegrationTestSuiteResults parseReportList( List<ReportTestSuite> reports )
{
Assert.assertTrue( "No reports!", reports.size() > 0 );
- int total = 0, errors = 0, failures = 0, skipped = 0;
+ int total = 0, errors = 0, failures = 0, skipped = 0, flakes = 0;
for ( ReportTestSuite report : reports )
{
total += report.getNumberOfTests();
errors += report.getNumberOfErrors();
failures += report.getNumberOfFailures();
skipped += report.getNumberOfSkipped();
+ flakes += report.getNumberOfFlakes();
}
- return new IntegrationTestSuiteResults( total, errors, failures, skipped );
+ return new IntegrationTestSuiteResults( total, errors, failures, skipped, flakes );
}
public static List<ReportTestSuite> extractReports( File[] testDirs )
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
index 8645027..f147281 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
@@ -22,7 +22,7 @@ package org.apache.maven.surefire.its.fixture;
public class IntegrationTestSuiteResults
{
- private int total, errors, failures, skipped;
+ private int total, errors, failures, skipped, flakes;
public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
{
@@ -32,6 +32,12 @@ public class IntegrationTestSuiteResults
this.skipped = skipped;
}
+ public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+ {
+ this(total, errors, failures, skipped);
+ this.flakes = flakes;
+ }
+
public int getTotal()
{
return total;
@@ -72,4 +78,14 @@ public class IntegrationTestSuiteResults
this.skipped = skipped;
}
+ public int getFlakes()
+ {
+ return flakes;
+ }
+
+ public void setFlakes( int flakes )
+ {
+ this.flakes = flakes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
index 2671879..14dfc47 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
@@ -125,6 +125,12 @@ public class OutputValidator
return this;
}
+ public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+ {
+ HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, flakes, baseDir );
+ return this;
+ }
+
public OutputValidator assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
{
HelperAssertions.assertIntegrationTestSuiteResults( total, errors, failures, skipped, baseDir );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
new file mode 100644
index 0000000..7482811
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>junit4-rerun-failing-tests</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for rerun failing tests in JUnit 4</name>
+
+
+ <properties>
+ <junitVersion>4.4</junitVersion>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junitVersion}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.5</source>
+ <target>1.5</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
new file mode 100644
index 0000000..264462c
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
@@ -0,0 +1,62 @@
+package junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class FlakyFirstTimeTest
+{
+ private static int failingCount = 0;
+
+ private static int errorCount = 0;
+
+
+ @Test
+ public void testFailingTestOne()
+ {
+ System.out.println( "Failing test" );
+ // This test will fail with only one retry, but will pass with two
+ if ( failingCount < 2 )
+ {
+ failingCount++;
+ Assert.fail( "Failing test" );
+ }
+ }
+
+ @Test
+ public void testErrorTestOne() throws Exception
+ {
+ System.out.println( "Error test" );
+ // This test will error out with only one retry, but will pass with two
+ if ( errorCount < 2 )
+ {
+ errorCount++;
+ throw new IllegalArgumentException("...");
+ }
+ }
+
+ @Test
+ public void testPassingTest() throws Exception
+ {
+ System.out.println( "Passing test" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
new file mode 100644
index 0000000..7cb0b57
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
@@ -0,0 +1,39 @@
+package junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class PassingTest
+{
+ @Test
+ public void testPassingTestOne()
+ {
+ System.out.println( "Passing test one" );
+ }
+
+ @Test
+ public void testPassingTestTwo() throws Exception
+ {
+ System.out.println( "Passing test two" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
new file mode 100644
index 0000000..52ce708
--- /dev/null
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
@@ -0,0 +1,109 @@
+package org.apache.maven.surefire.common.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.util.TestsToRun;
+import org.apache.maven.surefire.util.internal.StringUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.runner.notification.Failure;
+
+import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.isFailureInsideJUnitItself;
+
+/**
+ *
+ * Utility method used among all JUnit4 providers
+ *
+ * @author Qingzhou Luo
+ *
+ */
+public class JUnit4ProviderUtil
+{
+ /**
+ * Organize all the failures in previous run into a map between test classes and corresponding failing test methods
+ *
+ * @param allFailures all the failures in previous run
+ * @param testsToRun all the test classes
+ * @return a map between failing test classes and their corresponding failing test methods
+ */
+ public static Map<Class<?>, Set<String>> generateFailingTests( List<Failure> allFailures, TestsToRun testsToRun )
+ {
+ Map<Class<?>, Set<String>> testClassMethods = new HashMap<Class<?>, Set<String>>();
+
+ for ( Failure failure : allFailures )
+ {
+ if ( !isFailureInsideJUnitItself( failure ) )
+ {
+ // failure.getTestHeader() is in the format: method(class)
+ String[] testMethodClass = StringUtils.split( failure.getTestHeader(), "(" );
+ String testMethod = testMethodClass[0];
+ String testClass = StringUtils.split( testMethodClass[1], ")" )[0];
+ Class testClassObj = testsToRun.getClassByName( testClass );
+
+ if ( testClassObj == null )
+ {
+ continue;
+ }
+
+ Set<String> failingMethods = testClassMethods.get( testClassObj );
+ if ( failingMethods == null )
+ {
+ failingMethods = new HashSet<String>();
+ failingMethods.add( testMethod );
+ testClassMethods.put( testClassObj, failingMethods );
+ }
+ else
+ {
+ failingMethods.add( testMethod );
+ }
+ }
+ }
+ return testClassMethods;
+ }
+
+ /**
+ * Get the name of all test methods from a list of Failures
+ *
+ * @param allFailures the list of failures for a given test class
+ * @return the list of test method names
+ */
+ public static Set<String> generateFailingTests( List<Failure> allFailures )
+ {
+ Set<String> failingMethods = new HashSet<String>();
+
+ for ( Failure failure : allFailures )
+ {
+ if ( !isFailureInsideJUnitItself( failure ) )
+ {
+ // failure.getTestHeader() is in the format: method(class)
+ String testMethod = StringUtils.split( failure.getTestHeader(), "(" )[0];
+ failingMethods.add( testMethod );
+ }
+ }
+ return failingMethods;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
index c4b5b57..f21ab04 100644
--- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
@@ -209,7 +209,7 @@ public class JUnit4RunListener
}
}
- private static boolean isFailureInsideJUnitItself( Failure failure )
+ public static boolean isFailureInsideJUnitItself( Failure failure )
{
return failure.getDescription().getDisplayName().equals( "Test mechanism" );
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
new file mode 100644
index 0000000..5cca0e8
--- /dev/null
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
@@ -0,0 +1,35 @@
+package org.apache.maven.surefire.common.junit4;
+
+import org.junit.runner.notification.Failure;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test listener to record all the failures during one run
+ *
+ * @author Qingzhou Luo
+ */
+public class JUnitTestFailureListener
+ extends org.junit.runner.notification.RunListener
+{
+
+ List<Failure> allFailures = new ArrayList<Failure>();
+
+ @Override
+ public void testFailure( Failure failure )
+ throws java.lang.Exception
+ {
+ allFailures.add( failure );
+ }
+
+ public List<Failure> getAllFailures()
+ {
+ return allFailures;
+ }
+
+ public void reset()
+ {
+ allFailures.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
new file mode 100644
index 0000000..d21393a
--- /dev/null
+++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
@@ -0,0 +1,85 @@
+package org.apache.maven.surefire.common.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.maven.surefire.util.TestsToRun;
+import org.junit.runner.Description;
+import org.junit.runner.notification.Failure;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests;
+
+/**
+ * @author Qingzhou Luo
+ */
+public class JUnit4ProviderUtilTest
+ extends TestCase
+{
+ public void testGenerateFailingTests()
+ {
+ TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) );
+ List<Failure> failures = new ArrayList<Failure>( );
+
+ Description test1Description = Description.createTestDescription( T1.class, "testOne" );
+ Description test2Description = Description.createTestDescription( T1.class, "testTwo" );
+ Description test3Description = Description.createTestDescription( T2.class, "testThree" );
+ Description test4Description = Description.createTestDescription( T2.class, "testFour" );
+ Description test5Description = Description.createSuiteDescription( "Test mechanism" );
+
+ failures.add( new Failure( test1Description, new AssertionError() ) );
+ failures.add( new Failure( test2Description, new AssertionError() ) );
+ failures.add( new Failure( test3Description, new RuntimeException() ) );
+ failures.add( new Failure( test4Description, new AssertionError() ) );
+ failures.add( new Failure( test5Description, new RuntimeException() ) );
+
+ Map<Class<?>, Set<String>> result = generateFailingTests( failures, testsToRun );
+
+ assertEquals( 2, result.size() );
+ Set<String> resultForT1 = result.get( T1.class );
+ Set<String> resultForT2 = result.get( T2.class );
+
+ Set<String> expectedResultForT1 = new HashSet<String>();
+ expectedResultForT1.add( "testOne" );
+ expectedResultForT1.add( "testTwo" );
+ Set<String> expectedResultForT2 = new HashSet<String>();
+ expectedResultForT2.add( "testThree" );
+ expectedResultForT2.add( "testFour" );
+
+ assertEquals( expectedResultForT1, resultForT1 );
+ assertEquals( expectedResultForT2, resultForT2 );
+ }
+
+ class T1
+ {
+
+ }
+
+ class T2
+ {
+
+ }
+}
\ No newline at end of file