You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ag...@apache.org on 2014/07/31 12:47:41 UTC
[1/4] Add rerunFailingTestsCount option for maven surefire to rerun
failing tests immediately after they fail.
Repository: maven-surefire
Updated Branches:
refs/heads/master 1cdf49dc2 -> 2bdeeaf1a
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java
index f3fcc0b..3c2fa86 100644
--- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java
+++ b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
@@ -38,6 +39,7 @@ import org.apache.maven.surefire.group.parse.ParseException;
import org.junit.experimental.categories.Category;
import org.junit.runner.Description;
import org.junit.runner.manipulation.Filter;
+import sun.security.krb5.internal.crypto.Des;
/**
* @author Todd Lipcon
@@ -102,6 +104,11 @@ public class FilterFactory
return new MethodFilter( requestedTestMethod );
}
+ public Filter createFailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodMap )
+ {
+ return new FailingMethodFilter( failingClassMethodMap );
+ }
+
public Filter and( Filter filter1, Filter filter2 )
{
return new AndFilter( filter1, filter2 );
@@ -144,6 +151,56 @@ public class FilterFactory
}
}
+ // Only run test methods in the given input map, indexed by test class
+ private static class FailingMethodFilter
+ extends Filter
+ {
+ // Map from Class -> List of method names. Are the method names hashed to include the signature?
+ private final Map<Class<?>, Set<String>> failingClassMethodMap;
+
+ public FailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodMap )
+ {
+ this.failingClassMethodMap = failingClassMethodMap;
+ }
+
+ @Override
+ public boolean shouldRun( Description description )
+ {
+ return isDescriptionMatch( description );
+ }
+
+ private boolean isDescriptionMatch( Description description )
+ {
+ if ( description.getTestClass() == null || description.getMethodName() == null )
+ {
+ for ( Description childrenDescription : description.getChildren() )
+ {
+ if ( isDescriptionMatch( childrenDescription ) )
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ Set<String> testMethods = failingClassMethodMap.get( description.getTestClass() );
+ if ( testMethods == null )
+ {
+ return false;
+ }
+ else
+ {
+ return testMethods.contains( description.getMethodName() );
+ }
+ }
+
+ @Override
+ public String describe()
+ {
+ return "By failing class method";
+ }
+ }
+
private static class GroupMatcherCategoryFilter
extends Filter
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java
----------------------------------------------------------------------
diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java
index a03c7d2..03012bd 100644
--- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java
+++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java
@@ -19,14 +19,12 @@ package org.apache.maven.surefire.junit4;
* under the License.
*/
-import java.lang.reflect.Method;
-import java.util.Iterator;
-import java.util.List;
-
import org.apache.maven.shared.utils.io.SelectorUtils;
+import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil;
import org.apache.maven.surefire.common.junit4.JUnit4RunListener;
import org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory;
import org.apache.maven.surefire.common.junit4.JUnit4TestChecker;
+import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener;
import org.apache.maven.surefire.providerapi.AbstractProvider;
import org.apache.maven.surefire.providerapi.ProviderParameters;
import org.apache.maven.surefire.report.ConsoleOutputCapture;
@@ -43,12 +41,19 @@ import org.apache.maven.surefire.util.RunOrderCalculator;
import org.apache.maven.surefire.util.ScanResult;
import org.apache.maven.surefire.util.TestsToRun;
import org.apache.maven.surefire.util.internal.StringUtils;
-
import org.junit.runner.Request;
import org.junit.runner.Result;
import org.junit.runner.Runner;
+import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunNotifier;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
/**
* @author Kristian Rosenvold
*/
@@ -71,6 +76,8 @@ public class JUnit4Provider
private final ScanResult scanResult;
+ private final int rerunFailingTestsCount;
+
public JUnit4Provider( ProviderParameters booterParameters )
{
@@ -82,7 +89,7 @@ public class JUnit4Provider
createCustomListeners( booterParameters.getProviderProperties().getProperty( "listener" ) );
jUnit4TestChecker = new JUnit4TestChecker( testClassLoader );
requestedTestMethod = booterParameters.getTestRequest().getRequestedTestMethod();
-
+ rerunFailingTestsCount = booterParameters.getTestRequest().getRerunFailingTestsCount();
}
public RunResult invoke( Object forkTestSet )
@@ -108,7 +115,7 @@ public class JUnit4Provider
final ReporterFactory reporterFactory = providerParameters.getReporterFactory();
- final RunListener reporter = reporterFactory.createReporter();
+ RunListener reporter = reporterFactory.createReporter();
ConsoleOutputCapture.startCapture( (ConsoleOutputReceiver) reporter );
@@ -121,7 +128,7 @@ public class JUnit4Provider
for ( Class aTestsToRun : testsToRun )
{
- executeTestSet( aTestsToRun, reporter, runNotifer );
+ executeTestSet( aTestsToRun, reporter, runNotifer, null );
}
runNotifer.fireTestRunFinished( result );
@@ -129,28 +136,30 @@ public class JUnit4Provider
JUnit4RunListener.rethrowAnyTestMechanismFailures( result );
closeRunNotifer( jUnit4TestSetReporter, customRunListeners );
-
return reporterFactory.close();
}
- private void executeTestSet( Class<?> clazz, RunListener reporter, RunNotifier listeners )
+ private void executeTestSet( Class<?> clazz, RunListener reporter, RunNotifier listeners,
+ String[] failingTestMethods )
throws ReporterException, TestSetFailedException
{
final ReportEntry report = new SimpleReportEntry( this.getClass().getName(), clazz.getName() );
reporter.testSetStarting( report );
+
+
try
{
if ( !StringUtils.isBlank( this.requestedTestMethod ) )
{
String actualTestMethod = getMethod( clazz, this.requestedTestMethod );//add by rainLee
String[] testMethods = StringUtils.split( actualTestMethod, "+" );
- execute( clazz, listeners, testMethods );
+ executeWithRerun( clazz, listeners, testMethods );
}
else
{//the original way
- execute( clazz, listeners, null );
+ executeWithRerun( clazz, listeners, failingTestMethods );
}
}
catch ( TestSetFailedException e )
@@ -169,6 +178,34 @@ public class JUnit4Provider
}
}
+ private void executeWithRerun( Class<?> clazz, RunNotifier listeners, String[] testMethods )
+ throws TestSetFailedException
+ {
+ JUnitTestFailureListener failureListener = new JUnitTestFailureListener();
+ listeners.addListener( failureListener );
+
+ execute( clazz, listeners, testMethods );
+
+ // Rerun failing tests if rerunFailingTestsCount is larger than 0
+ int rerunCount = this.rerunFailingTestsCount;
+ if ( rerunCount > 0 )
+ {
+ for ( int i = 0; i < rerunCount; i++ )
+ {
+ List<Failure> failures = failureListener.getAllFailures();
+ if ( failures.size() == 0 )
+ {
+ break;
+ }
+
+ Set<String> methodsSet = JUnit4ProviderUtil.generateFailingTests( failureListener.getAllFailures() );
+ String[] methods = methodsSet.toArray(new String[methodsSet.size()]);
+ failureListener.reset();
+ execute( clazz, listeners, methods );
+ }
+ }
+ }
+
private RunNotifier getRunNotifer( org.junit.runner.notification.RunListener main, Result result,
List<org.junit.runner.notification.RunListener> others )
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java
----------------------------------------------------------------------
diff --git a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java
index 147a5c1..bbca2d3 100644
--- a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java
+++ b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderTest.java
@@ -19,11 +19,18 @@ package org.apache.maven.surefire.junit4;
* under the License.
*/
-import java.util.Properties;
+import junit.framework.TestCase;
import org.apache.maven.surefire.booter.BaseProviderFactory;
import org.apache.maven.surefire.testset.TestRequest;
+import org.apache.maven.surefire.util.TestsToRun;
+import org.junit.runner.Description;
+import org.junit.runner.notification.Failure;
-import junit.framework.TestCase;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
/**
* @author Kristian Rosenvold
@@ -33,10 +40,15 @@ public class JUnit4ProviderTest
{
public void testCreateProvider()
{
+ assertNotNull( getJUnit4Provider() );
+ }
+
+ private JUnit4Provider getJUnit4Provider()
+ {
BaseProviderFactory providerParameters = new BaseProviderFactory( null, Boolean.TRUE );
providerParameters.setProviderProperties( new Properties() );
providerParameters.setClassLoaders( this.getClass().getClassLoader() );
providerParameters.setTestRequest( new TestRequest( null, null, null ) );
- assertNotNull( new JUnit4Provider( providerParameters ) );
+ return new JUnit4Provider( providerParameters );
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java
----------------------------------------------------------------------
diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java
index 04f1673..8cbf0cd 100644
--- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java
+++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java
@@ -19,12 +19,17 @@ package org.apache.maven.surefire.junitcore;
* under the License.
*/
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil;
import org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory;
+import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener;
import org.apache.maven.surefire.common.junit48.FilterFactory;
import org.apache.maven.surefire.common.junit48.JUnit48Reflector;
import org.apache.maven.surefire.common.junit48.JUnit48TestChecker;
@@ -43,7 +48,11 @@ import org.apache.maven.surefire.util.ScanResult;
import org.apache.maven.surefire.util.ScannerFilter;
import org.apache.maven.surefire.util.TestsToRun;
import org.apache.maven.surefire.util.internal.StringUtils;
+import org.junit.internal.runners.statements.Fail;
import org.junit.runner.manipulation.Filter;
+import org.junit.runner.notification.Failure;
+
+import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.isFailureInsideJUnitItself;
/**
* @author Kristian Rosenvold
@@ -72,6 +81,8 @@ public class JUnitCoreProvider
private final ScanResult scanResult;
+ private final int rerunFailingTestsCount;
+
public JUnitCoreProvider( ProviderParameters providerParameters )
{
this.providerParameters = providerParameters;
@@ -81,6 +92,7 @@ public class JUnitCoreProvider
this.jUnitCoreParameters = new JUnitCoreParameters( providerParameters.getProviderProperties() );
this.scannerFilter = new JUnit48TestChecker( testClassLoader );
this.requestedTestMethod = providerParameters.getTestRequest().getRequestedTestMethod();
+ this.rerunFailingTestsCount = providerParameters.getTestRequest().getRerunFailingTestsCount();
customRunListeners =
JUnit4RunListenerFactory.createCustomListeners( providerParameters.getProviderProperties().getProperty( "listener" ) );
@@ -131,7 +143,33 @@ public class JUnitCoreProvider
org.junit.runner.notification.RunListener jUnit4RunListener = getRunListener( reporterFactory, consoleLogger );
customRunListeners.add( 0, jUnit4RunListener );
+
+ // Add test failure listener
+ JUnitTestFailureListener testFailureListener = new JUnitTestFailureListener();
+ customRunListeners.add( 0, testFailureListener );
+
JUnitCoreWrapper.execute( testsToRun, jUnitCoreParameters, customRunListeners, filter );
+
+ // Rerun failing tests if rerunFailingTestsCount is larger than 0
+ int rerunCount = this.rerunFailingTestsCount;
+ if ( rerunCount > 0 )
+ {
+ for ( int i = 0; i < rerunCount; i++ )
+ {
+ List<Failure> failures = testFailureListener.getAllFailures();
+ if ( failures.size() == 0 )
+ {
+ break;
+ }
+ Map<Class<?>, Set<String>> failingTests =
+ JUnit4ProviderUtil.generateFailingTests( failures, testsToRun );
+ testFailureListener.reset();
+ final FilterFactory filterFactory = new FilterFactory( testClassLoader );
+ Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests );
+ JUnitCoreWrapper.execute( testsToRun, jUnitCoreParameters, customRunListeners,
+ filterFactory.and( filter, failingMethodsFilter ) );
+ }
+ }
return reporterFactory.close();
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java
----------------------------------------------------------------------
diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java
index f41554d..24fb2da 100644
--- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java
+++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/ConcurrentRunListenerTest.java
@@ -338,7 +338,7 @@ public class ConcurrentRunListenerTest
TestSuite suite = new TestSuite();
suite.addTest( new Junit3OddTest1( "testMe" ) );
- suite.addTest( new Junit3OddTest1( "testMe" ) );
+ suite.addTest( new Junit3OddTest1( "testMe2" ) );
return suite;
}
@@ -361,7 +361,7 @@ public class ConcurrentRunListenerTest
{
TestSuite suite = new TestSuite();
- suite.addTest( new Junit3WithNestedSuite( "testMe2" ) );
+ suite.addTest( new Junit3WithNestedSuite( "testMe" ) );
suite.addTest( new Junit3WithNestedSuite( "testMe2" ) );
suite.addTestSuite( Junit3Tc2.class );
return suite;
@@ -388,7 +388,6 @@ public class ConcurrentRunListenerTest
private void assertReporter( RunStatistics result, int success, int ignored, int failure, String message )
{
assertEquals( message, success, result.getCompletedCount() );
- assertEquals( message, failure, result.getFailureSources().size() );
assertEquals( message, ignored, result.getSkipped() );
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java
----------------------------------------------------------------------
diff --git a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java
index 2bd6513..989eda4 100644
--- a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java
+++ b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestSuite.java
@@ -35,6 +35,8 @@ public class ReportTestSuite
private int numberOfSkipped;
+ private int numberOfFlakes;
+
private Integer numberOfTests;
private String name;
@@ -80,6 +82,16 @@ public class ReportTestSuite
this.numberOfSkipped = numberOfSkipped;
}
+ public int getNumberOfFlakes()
+ {
+ return numberOfFlakes;
+ }
+
+ public void setNumberOfFlakes( int numberOfFlakes )
+ {
+ this.numberOfFlakes = numberOfFlakes;
+ }
+
public int getNumberOfTests()
{
if ( numberOfTests != null )
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
----------------------------------------------------------------------
diff --git a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
index e54443d..72afd44 100644
--- a/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
+++ b/surefire-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
@@ -218,6 +218,10 @@ public class TestSuiteXmlParser
testCase.addFailure( message != null ? message : "skipped", "skipped" );
currentSuite.setNumberOfSkipped( 1 + currentSuite.getNumberOfSkipped() );
}
+ else if ( "flakyFailure".equals( qName ) || "flakyError".equals( qName ) )
+ {
+ currentSuite.setNumberOfFlakes( 1 + currentSuite.getNumberOfFlakes() );
+ }
else if ( "failsafe-summary".equals( qName ) )
{
valid = false;
[2/4] Add rerunFailingTestsCount option for maven surefire to rerun
failing tests immediately after they fail.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
deleted file mode 100644
index 5a74c45..0000000
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXMLReporterTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-package org.apache.maven.plugin.surefire.report;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import junit.framework.AssertionFailedError;
-import junit.framework.TestCase;
-
-import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter;
-import org.apache.maven.shared.utils.StringUtils;
-import org.apache.maven.shared.utils.xml.Xpp3Dom;
-import org.apache.maven.shared.utils.xml.Xpp3DomBuilder;
-import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter;
-import org.apache.maven.surefire.report.ReportEntry;
-import org.apache.maven.surefire.report.SimpleReportEntry;
-import org.apache.maven.surefire.report.StackTraceWriter;
-
-@SuppressWarnings( "ResultOfMethodCallIgnored" )
-public class StatelessXMLReporterTest
- extends TestCase
-{
-
- private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false );
-
- private ReportEntry reportEntry;
-
- private TestSetStats stats;
-
- private File expectedReportFile;
-
- protected void setUp()
- throws Exception
- {
- super.setUp();
- reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest",
- new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 );
- stats = new TestSetStats( false, true );
- }
-
- @Override protected void tearDown()
- throws Exception
- {
- super.tearDown();
-
- if ( expectedReportFile != null )
- {
- expectedReportFile.delete();
- }
- }
-
- public void testFileNameWithoutSuffix()
- {
- File reportDir = new File( "." );
- String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest";
- reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
- WrappedReportEntry testSetReportEntry =
- new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
- stats.testSucceeded( testSetReportEntry );
- reporter.testSetCompleted( testSetReportEntry, stats );
-
- expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
- assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist",
- expectedReportFile.exists() );
- }
-
-
- public void testAllFieldsSerialized()
- throws IOException
- {
- File reportDir = new File( "." );
- String testName = "aTestMethod";
- String testName2 = "bTestMethod";
- reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
- WrappedReportEntry testSetReportEntry =
- new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
- expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
-
- stats.testSucceeded( testSetReportEntry );
- StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" );
- Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" );
- String stdOutPrefix;
- String stdErrPrefix;
- if ( defaultCharsetSupportsSpecialChar() )
- {
- stdErrPrefix = "std-\u0115rr";
- stdOutPrefix = "st]]>d-o\u00DCt";
- }
- else
- {
- stdErrPrefix = "std-err";
- stdOutPrefix = "st]]>d-out";
- }
-
- byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes();
- stdOut.write( stdOutBytes, 0, stdOutBytes.length );
-
- Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" );
-
-
- byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes();
- stdErr.write( stdErrBytes, 0, stdErrBytes.length );
- WrappedReportEntry t2 =
- new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), testName2, stackTraceWriter, 13 ),
- ReportEntryType.error, 13, stdOut, stdErr );
-
- stats.testSucceeded( t2 );
- StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false );
- reporter.testSetCompleted( testSetReportEntry, stats );
-
- FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
-
- Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") );
- assertEquals( "testsuite", testSuite.getName() );
- Xpp3Dom properties = testSuite.getChild( "properties" );
- assertEquals( System.getProperties().size(), properties.getChildCount() );
- Xpp3Dom child = properties.getChild( 1 );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
-
- Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
- Xpp3Dom tca = testcase[0];
- assertEquals( testName, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5
- assertEquals( "0.012", tca.getAttribute( "time" ) );
- assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) );
-
- Xpp3Dom tcb = testcase[1];
- assertEquals( testName2, tcb.getAttribute( "name" ) );
- assertEquals( "0.013", tcb.getAttribute( "time" ) );
- assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) );
- Xpp3Dom errorNode = tcb.getChild( "error" );
- assertNotNull( errorNode );
- assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
- assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
- assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() );
-
-
- assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() );
- }
-
- private boolean defaultCharsetSupportsSpecialChar()
- {
- // some charsets are not able to deal with \u0115 on both ways of the conversion
- return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) );
- }
-
- class Inner
- {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
new file mode 100644
index 0000000..c6c1d15
--- /dev/null
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
@@ -0,0 +1,290 @@
+package org.apache.maven.plugin.surefire.report;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.AssertionFailedError;
+import junit.framework.TestCase;
+import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktraceWriter;
+import org.apache.maven.shared.utils.StringUtils;
+import org.apache.maven.shared.utils.xml.Xpp3Dom;
+import org.apache.maven.shared.utils.xml.Xpp3DomBuilder;
+import org.apache.maven.surefire.report.LegacyPojoStackTraceWriter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.SimpleReportEntry;
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+@SuppressWarnings( "ResultOfMethodCallIgnored" )
+public class StatelessXmlReporterTest
+ extends TestCase
+{
+
+ private StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 );
+
+ private ReportEntry reportEntry;
+
+ private TestSetStats stats;
+
+ private TestSetStats rerunStats;
+
+ private File expectedReportFile;
+
+ private final static String TEST_ONE = "aTestMethod";
+ private final static String TEST_TWO = "bTestMethod";
+ private final static String TEST_THREE = "cTestMethod";
+
+ protected void setUp()
+ throws Exception
+ {
+ super.setUp();
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), "StatelessXMLReporterTest",
+ new LegacyPojoStackTraceWriter( "", "", new AssertionFailedError() ), 17 );
+ stats = new TestSetStats( false, true );
+ rerunStats = new TestSetStats( false, true );
+ reporter.cleanTestHistoryMap();
+ }
+
+ @Override protected void tearDown()
+ throws Exception
+ {
+ super.tearDown();
+
+ if ( expectedReportFile != null )
+ {
+ expectedReportFile.delete();
+ }
+ }
+
+ public void testFileNameWithoutSuffix()
+ {
+ File reportDir = new File( "." );
+ String testName = "org.apache.maven.plugin.surefire.report.StatelessXMLReporterTest";
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), testName, 12 );
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ stats.testSucceeded( testSetReportEntry );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+
+ expectedReportFile = new File( reportDir, "TEST-" + testName + ".xml" );
+ assertTrue( "Report file (" + expectedReportFile.getAbsolutePath() + ") doesn't exist",
+ expectedReportFile.exists() );
+ }
+
+
+ public void testAllFieldsSerialized()
+ throws IOException
+ {
+ File reportDir = new File( "." );
+
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 );
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" );
+
+ stats.testSucceeded( testSetReportEntry );
+ StackTraceWriter stackTraceWriter = new DeserializedStacktraceWriter( "A fud msg", "trimmed", "fail at foo" );
+ Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds" );
+ String stdOutPrefix;
+ String stdErrPrefix;
+ if ( defaultCharsetSupportsSpecialChar() )
+ {
+ stdErrPrefix = "std-\u0115rr";
+ stdOutPrefix = "st]]>d-o\u00DCt";
+ }
+ else
+ {
+ stdErrPrefix = "std-err";
+ stdOutPrefix = "st]]>d-out";
+ }
+
+ byte[] stdOutBytes = (stdOutPrefix + "<null>!\u0020\u0000\u001F").getBytes();
+ stdOut.write( stdOutBytes, 0, stdOutBytes.length );
+
+ Utf8RecodingDeferredFileOutputStream stdErr = new Utf8RecodingDeferredFileOutputStream( "fds" );
+
+
+ byte[] stdErrBytes = (stdErrPrefix + "?&-&£\u0020\u0000\u001F").getBytes();
+ stdErr.write( stdErrBytes, 0, stdErrBytes.length );
+ WrappedReportEntry t2 =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriter, 13 ),
+ ReportEntryType.error, 13, stdOut, stdErr );
+
+ stats.testSucceeded( t2 );
+ StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 0 );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+
+ FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
+
+ Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8") );
+ assertEquals( "testsuite", testSuite.getName() );
+ Xpp3Dom properties = testSuite.getChild( "properties" );
+ assertEquals( System.getProperties().size(), properties.getChildCount() );
+ Xpp3Dom child = properties.getChild( 1 );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+
+ Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
+ Xpp3Dom tca = testcase[0];
+ assertEquals( TEST_ONE, tca.getAttribute( "name" ) ); // Hopefully same order on jdk5
+ assertEquals( "0.012", tca.getAttribute( "time" ) );
+ assertEquals( this.getClass().getName(), tca.getAttribute( "classname" ) );
+
+ Xpp3Dom tcb = testcase[1];
+ assertEquals( TEST_TWO, tcb.getAttribute( "name" ) );
+ assertEquals( "0.013", tcb.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), tcb.getAttribute( "classname" ) );
+ Xpp3Dom errorNode = tcb.getChild( "error" );
+ assertNotNull( errorNode );
+ assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
+ assertEquals( stdOutPrefix + "<null>! &#0;&#31;", tcb.getChild( "system-out" ).getValue() );
+
+
+ assertEquals( stdErrPrefix + "?&-&£ &#0;&#31;", tcb.getChild( "system-err" ).getValue() );
+ }
+
+ public void testOutputRerunFlakyFailure()
+ throws IOException
+ {
+ File reportDir = new File( "." );
+ reportEntry = new SimpleReportEntry( this.getClass().getName(), TEST_ONE, 12 );
+
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( reportEntry, ReportEntryType.success, 12, null, null );
+ expectedReportFile = new File( reportDir, "TEST-" + TEST_ONE + ".xml" );
+
+ stats.testSucceeded( testSetReportEntry );
+ StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed",
+ "fail at foo" );
+ StackTraceWriter stackTraceWriterTwo =
+ new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" );
+
+ String firstRunOut = "first run out";
+ String firstRunErr = "first run err";
+ String secondRunOut = "second run out";
+ String secondRunErr = "second run err";
+
+ WrappedReportEntry testTwoFirstError =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterOne, 5 ),
+ ReportEntryType.error, 5, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ WrappedReportEntry testTwoSecondError =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_TWO, stackTraceWriterTwo, 13 ),
+ ReportEntryType.error, 13, createStdOutput( secondRunOut ),
+ createStdOutput( secondRunErr ) );
+
+ WrappedReportEntry testThreeFirstRun =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterOne, 13 ),
+ ReportEntryType.failure, 13, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ WrappedReportEntry testThreeSecondRun =
+ new WrappedReportEntry( new SimpleReportEntry( Inner.class.getName(), TEST_THREE, stackTraceWriterTwo, 2 ),
+ ReportEntryType.success, 2, createStdOutput( secondRunOut ),
+ createStdOutput( secondRunErr ) );
+
+ stats.testSucceeded( testTwoFirstError );
+ stats.testSucceeded( testThreeFirstRun );
+ rerunStats.testSucceeded( testTwoSecondError );
+ rerunStats.testSucceeded( testThreeSecondRun );
+
+ StatelessXmlReporter reporter = new StatelessXmlReporter( new File( "." ), null, false, 1 );
+ reporter.testSetCompleted( testSetReportEntry, stats );
+ reporter.testSetCompleted( testSetReportEntry, rerunStats );
+
+ FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
+
+ Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, "UTF-8" ) );
+ assertEquals( "testsuite", testSuite.getName() );
+ // 0.019 = 0.012 + 0.005 +0.002
+ assertEquals( "0.019", testSuite.getAttribute( "time" ) );
+ Xpp3Dom properties = testSuite.getChild( "properties" );
+ assertEquals( System.getProperties().size(), properties.getChildCount() );
+ Xpp3Dom child = properties.getChild( 1 );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+
+ Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
+ Xpp3Dom testCaseOne = testcase[0];
+ assertEquals( TEST_ONE, testCaseOne.getAttribute( "name" ) );
+ assertEquals( "0.012", testCaseOne.getAttribute( "time" ) );
+ assertEquals( this.getClass().getName(), testCaseOne.getAttribute( "classname" ) );
+
+ Xpp3Dom testCaseTwo = testcase[1];
+ assertEquals( TEST_TWO, testCaseTwo.getAttribute( "name" ) );
+ // Run time for a rerun failing test is the run time of the first run
+ assertEquals( "0.005", testCaseTwo.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), testCaseTwo.getAttribute( "classname" ) );
+ Xpp3Dom errorNode = testCaseTwo.getChild( "error" );
+ Xpp3Dom rerunErrorNode = testCaseTwo.getChild( "rerunError" );
+ assertNotNull( errorNode );
+ assertNotNull( rerunErrorNode );
+
+ assertEquals( "A fud msg", errorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo", errorNode.getAttribute( "type" ) );
+
+ // Check rerun error node contains all the information
+ assertEquals( firstRunOut, testCaseTwo.getChild( "system-out" ).getValue() );
+ assertEquals( firstRunErr, testCaseTwo.getChild( "system-err" ).getValue() );
+ assertEquals( secondRunOut, rerunErrorNode.getChild( "system-out" ).getValue() );
+ assertEquals( secondRunErr, rerunErrorNode.getChild( "system-err" ).getValue() );
+ assertEquals( "A fud msg two", rerunErrorNode.getAttribute( "message" ) );
+ assertEquals( "fail at foo two", rerunErrorNode.getAttribute( "type" ) );
+
+ // Check flaky failure node
+ Xpp3Dom testCaseThree = testcase[2];
+ assertEquals( TEST_THREE, testCaseThree.getAttribute( "name" ) );
+ // Run time for a flaky test is the run time of the first successful run
+ assertEquals( "0.002", testCaseThree.getAttribute( "time" ) );
+ assertEquals( Inner.class.getName(), testCaseThree.getAttribute( "classname" ) );
+ Xpp3Dom flakyFailureNode = testCaseThree.getChild( "flakyFailure" );
+ assertNotNull( flakyFailureNode );
+ assertEquals( firstRunOut, flakyFailureNode.getChild( "system-out" ).getValue() );
+ assertEquals( firstRunErr, flakyFailureNode.getChild( "system-err" ).getValue() );
+ // system-out and system-err should not be present for flaky failures
+ assertNull( testCaseThree.getChild( "system-out" ) );
+ assertNull( testCaseThree.getChild( "system-err" ) );
+ }
+
+ private boolean defaultCharsetSupportsSpecialChar()
+ {
+ // some charsets are not able to deal with \u0115 on both ways of the conversion
+ return "\u0115\u00DC".equals( new String( "\u0115\u00DC".getBytes() ) );
+ }
+
+ private Utf8RecodingDeferredFileOutputStream createStdOutput( String content )
+ throws IOException
+ {
+ Utf8RecodingDeferredFileOutputStream stdOut = new Utf8RecodingDeferredFileOutputStream( "fds2" );
+ stdOut.write( content.getBytes(), 0, content.length() );
+ return stdOut;
+ }
+
+ class Inner
+ {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
index ca6c469..acb17cc 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
@@ -1,7 +1,5 @@
package org.apache.maven.surefire.report;
-import java.util.Collection;
-
import junit.framework.TestCase;
/*
@@ -24,85 +22,14 @@ import junit.framework.TestCase;
public class RunStatisticsTest
extends TestCase
{
- private static final String Method = "AClass#AMethod";
-
- private static final String DUMMY_ERROR_SOURCE = Method + " RuntimeException";
-
- private static final String DUMMY_FAILURE_SOURCE = "dummy failure source";
-
- private static final String DUMMY_MESSAGE = "foo";
-
- public void testAddErrorSourceWithThrowableMessage()
- {
- RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE );
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable );
- assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE );
- }
-
- public void testAddErrorSourceWithoutThrowable()
- {
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( null );
- assertRunStatisticsHasErrorSource( statistics, Method );
- }
-
- public void testAddErrorSourceWithThrowableWithoutMessage()
- {
- RuntimeException throwable = new RuntimeException();
- RunStatistics statistics = createRunStatisticsAndAddErrorSourceWithThrowable( throwable );
- assertRunStatisticsHasErrorSource( statistics, DUMMY_ERROR_SOURCE );
- }
-
- public void testAddFailureSourceWithThrowableMessage()
+ public void testSetRunStatistics()
{
- RuntimeException throwable = new RuntimeException( DUMMY_MESSAGE );
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable );
- assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE + " " + DUMMY_MESSAGE );
- }
-
- public void testAddFailureSourceWithoutThrowable()
- {
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( null );
- assertRunStatisticsHasFailureSource( statistics, Method );
- }
-
- public void testAddFailureSourceWithThrowableWithoutMessage()
- {
- RuntimeException throwable = new RuntimeException();
- RunStatistics statistics = createRunStatisticsAndAddFailureSourceWithThrowable( throwable );
- assertRunStatisticsHasFailureSource( statistics, DUMMY_ERROR_SOURCE );
- }
-
- private RunStatistics createRunStatisticsAndAddErrorSourceWithThrowable( Throwable throwable )
- {
- StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable );
RunStatistics statistics = new RunStatistics();
- statistics.addErrorSource( stackTraceWriter );
-
- return statistics;
- }
-
- private RunStatistics createRunStatisticsAndAddFailureSourceWithThrowable( Throwable throwable )
- {
- StackTraceWriter stackTraceWriter = new LegacyPojoStackTraceWriter( "AClass", "AMethod", throwable );
- RunStatistics statistics = new RunStatistics();
- statistics.addFailureSource( stackTraceWriter );
-
- return statistics;
- }
-
- private void assertRunStatisticsHasErrorSource( RunStatistics statistics, String expectedErrorSource )
- {
- Collection errorSources = statistics.getErrorSources();
- assertNotNull( "No error sources.", errorSources );
- assertEquals( "Wrong number of error sources.", 1, errorSources.size() );
- assertEquals( "Wrong error sources.", expectedErrorSource, errorSources.iterator().next() );
- }
-
- private void assertRunStatisticsHasFailureSource( RunStatistics statistics, String expectedFailureSource )
- {
- Collection failureSources = statistics.getFailureSources();
- assertNotNull( "No failure sources.", failureSources );
- assertEquals( "Wrong number of failure sources.", 1, failureSources.size() );
- assertEquals( "Wrong failure sources.", expectedFailureSource, failureSources.iterator().next() );
+ statistics.set( 10, 5, 2, 1, 2 );
+ assertEquals( 10, statistics.getCompletedCount() );
+ assertEquals( 5, statistics.getErrors() );
+ assertEquals( 2, statistics.getFailures() );
+ assertEquals( 1, statistics.getSkipped() );
+ assertEquals( 2, statistics.getFlakes() );
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index 131ad74..0b8d030 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -187,6 +187,18 @@ public class SurefirePlugin
@Parameter( property = "surefire.useManifestOnlyJar", defaultValue = "true" )
private boolean useManifestOnlyJar;
+ /**
+ * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
+ * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
+ * However, all the failing attempts will be recorded.
+ */
+ @Parameter( property = "surefire.rerunFailingTestsCount", defaultValue = "0" )
+ protected int rerunFailingTestsCount;
+
+ protected int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
protected void handleSummary( RunResult summary, Exception firstForkException )
throws MojoExecutionException, MojoFailureException
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
new file mode 100644
index 0000000..42cc98c
--- /dev/null
+++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
@@ -0,0 +1,139 @@
+ ------
+ Rerun failing tests
+ ------
+ Qingzhou Luo
+ ------
+ 2014-06-27
+ ------
+
+ ~~ Licensed to the Apache Software Foundation (ASF) under one
+ ~~ or more contributor license agreements. See the NOTICE file
+ ~~ distributed with this work for additional information
+ ~~ regarding copyright ownership. The ASF licenses this file
+ ~~ to you under the Apache License, Version 2.0 (the
+ ~~ "License"); you may not use this file except in compliance
+ ~~ with the License. You may obtain a copy of the License at
+ ~~
+ ~~ http://www.apache.org/licenses/LICENSE-2.0
+ ~~
+ ~~ Unless required by applicable law or agreed to in writing,
+ ~~ software distributed under the License is distributed on an
+ ~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~~ KIND, either express or implied. See the License for the
+ ~~ specific language governing permissions and limitations
+ ~~ under the License.
+
+ ~~ NOTE: For help with the syntax of this file, see:
+ ~~ http://maven.apache.org/doxia/references/apt-format.html
+
+Rerun Failing Tests
+
+#{if}(${project.artifactId}=="maven-surefire-plugin")
+ During development, you may re-run failing tests because they are flaky.
+ To use this feature through Maven surefire, set the <<<rerunFailingTestsCount>>> property to be a value larger than 0.
+ Tests will be run until they pass or the number of reruns has been exhausted.
+
+ << NOTE : This feature is supported only for JUnit 4.x. >>
+
+
++---+
+mvn -DrerunFailingTestsCount=2 test
++---+
+
+ If <<<rerunFailingTestsCount>>> is set to a value smaller than or euqal to 0, then it will be ignored.
+
+* Output flaky re-run information on the screen
+
+ When <<<rerunFailingTestsCount>>> is set to a value larger than 0 and the test fails,
+ then it will be re-run and each run information will also be output. Each run with its number and trimmed stack trace
+ will be output.
+
+ If the test passes in its first run, then the output on the screen will be identical to the case where
+ <<<rerunFailingTestsCount>>> is not used.
+
+ It the test fails in the first run, then there are two possible cases:
+
+ 1) The test passes in one of its re-runs: the last run will be marked as PASS
+
+ For example, a test passed in its second run will output on the screen:
+
++---+
+ Run 1: ...
+ Run 2: PASS
++---+
+
+ Then this test will be counted as a flaky test. The build will be successful, but in the end of the summary of all
+ tests run, the number of flaky tests will be output on the screen, for example:
+
++---+
+ Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Flakes: 1
++---+
+
+ 2) The test fails in all of the re-runs:
+
+ For example, a test failied in all the three runs, then all the runs will be output on the screen:
+
++---+
+ Run 1: ...
+ Run 2: ...
+ Run 3: ...
++---+
+
+ Then this build will be marked as failure. The type of the failure (failed test or test in error) of this test
+ depends on its first failure.
+
+* Output flaky re-run information in test report xml
+
+ When <<<rerunFailingTestsCount>>> is set to a value larger than 0, the output xml of test report will also be extended
+ to include information of each re-run.
+
+ If the test passes in its first run, then the output xml report will be identical to the case where
+ <<<rerunFailingTestsCount>>> is not used.
+
+ It the test fails in the first run, then there are also two possible cases.
+
+ 1) The test passes in one of its re-runs:
+
+ <<<flakyFailure>>> and <<<flakyError>>> elements will be used in the generated xml report to include information
+ of each failing re-runs. <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>>
+ or <<<flakyError>>> to include information of System.out and System.err output. The original <<<system-out>>>
+ and <<<system-err>>> elements will be retained on the top level under <<<testcase>>> for the last successful run.
+
+ For example:
+
++---+
+<testcase name=".." classname=".." time="0.1">
+ <flakyFailure message="" type=""> flaky failure stack trace
+ <system-out> flaky failure </system-out>
+ </flakyFailure>
+ <system-out> success </system-out>
+</testcase>
++---+
+
+ In the xml report, the running time of a flaky test will be the running time of the <<last successful run>>.
+
+ 2) The test fails in all of the re-runs:
+
+ <<<failure>>> and <<<error>>> elements will still be used in the generated xml report to include information
+ for the first failing run, the same as without using <<<rerunFailingTests>>>. <<<rerunFailure>>> and <<<rerunError>>>
+ elements will be used in the generated xml report to include information of each <<subsequent>> failing re-runs.
+ <<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>> or <<<flakyError>>> to include
+ information of System.out and System.err output. The original <<<system-out>>> and <<<system-err>>> elements will be
+ retained on the top level under <<<testcase>>> for the first original failing run.
+
+ For example:
+
++---+
+<testcase name=".." classname=".." time="0.1">
+ <failure message="" type=""> first failure stack trace </failure>
+ <system-out> first failure </system-out>
+ <rerunFailure message="" type=""> rerun failure stack trace
+ <system-out> rerun failure </system-out>
+ </rerunFailure>
+</testcase>
++---+
+
+ In the xml report, the running time of a failing test with re-runs will be the running time of the
+ <<first failing run>>.
+
+#{end}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-plugin/src/site/site.xml
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/site/site.xml b/maven-surefire-plugin/src/site/site.xml
index 3724c87..d98d763 100644
--- a/maven-surefire-plugin/src/site/site.xml
+++ b/maven-surefire-plugin/src/site/site.xml
@@ -43,6 +43,7 @@
<item name="Skipping Tests" href="examples/skipping-test.html"/>
<item name="Inclusions and Exclusions of Tests" href="examples/inclusion-exclusion.html"/>
<item name="Running a Single Test" href="examples/single-test.html"/>
+ <item name="Re-run Failing Tests" href="examples/rerun-failing-tests.html"/>
<item name="Class Loading and Forking" href="examples/class-loading.html"/>
<item name="Debugging Tests" href="examples/debugging.html"/>
<item name="Using System Properties" href="examples/system-properties.html"/>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
index 54a6b5b..2e4bee4 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -152,11 +152,11 @@ public class SurefireReflector
{
return null;
}
- Class[] arguments = { List.class, File.class, String.class, String.class };
+ Class[] arguments = { List.class, File.class, String.class, String.class, int.class };
Constructor constructor = ReflectionUtils.getConstructor( this.testRequest, arguments );
- return ReflectionUtils.newInstance( constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(),
+ return ReflectionUtils.newInstance(constructor, new Object[]{ suiteDefinition.getSuiteXmlFiles(),
suiteDefinition.getTestSourceDirectory(), suiteDefinition.getRequestedTest(),
- suiteDefinition.getRequestedTestMethod() } );
+ suiteDefinition.getRequestedTestMethod(), suiteDefinition.getRerunFailingTestsCount() });
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
index 595717b..83d74f6 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/suite/RunResult.java
@@ -53,6 +53,8 @@ public class RunResult
private final int skipped;
+ private final int flakes;
+
private final String failure;
private final boolean timeout;
@@ -85,14 +87,26 @@ public class RunResult
this( completedCount, errors, failures, skipped, null, false );
}
+ public RunResult( int completedCount, int errors, int failures, int skipped, int flakes )
+ {
+ this( completedCount, errors, failures, skipped, flakes, null, false );
+ }
+
public RunResult( int completedCount, int errors, int failures, int skipped, String failure, boolean timeout )
{
+ this( completedCount, errors, failures, skipped, 0, failure, timeout );
+ }
+
+ public RunResult( int completedCount, int errors, int failures, int skipped, int flakes, String failure,
+ boolean timeout )
+ {
this.completedCount = completedCount;
this.errors = errors;
this.failures = failures;
this.skipped = skipped;
this.failure = failure;
this.timeout = timeout;
+ this.flakes = flakes;
}
private static String getStackTrace( Exception e )
@@ -117,6 +131,11 @@ public class RunResult
return errors;
}
+ public int getFlakes()
+ {
+ return flakes;
+ }
+
public int getFailures()
{
return failures;
@@ -176,7 +195,8 @@ public class RunResult
int fail = getFailures() + other.getFailures();
int ign = getSkipped() + other.getSkipped();
int err = getErrors() + other.getErrors();
- return new RunResult( completed, err, fail, ign, failureMessage, timeout );
+ int flakes = getFlakes() + other.getFlakes();
+ return new RunResult( completed, err, fail, ign, flakes, failureMessage, timeout );
}
public static RunResult noTestsRun()
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
index 190b71b..a237bbe 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/testset/TestRequest.java
@@ -36,6 +36,8 @@ public class TestRequest
private final String requestedTest;
+ private final int rerunFailingTestsCount;
+
/**
* @since 2.7.3
*/
@@ -51,10 +53,17 @@ public class TestRequest
*/
public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod )
{
+ this( createFiles( suiteXmlFiles ), testSourceDirectory, requestedTest, requestedTestMethod, 0 );
+ }
+
+ public TestRequest( List suiteXmlFiles, File testSourceDirectory, String requestedTest, String requestedTestMethod,
+ int rerunFailingTestsCount )
+ {
this.suiteXmlFiles = createFiles( suiteXmlFiles );
this.testSourceDirectory = testSourceDirectory;
this.requestedTest = requestedTest;
this.requestedTestMethod = requestedTestMethod;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
/**
@@ -98,6 +107,16 @@ public class TestRequest
return requestedTestMethod;
}
+ /**
+ * How many times to rerun failing tests, issued with -Dsurefire.rerunFailingTestsCount from the command line.
+ *
+ * @return The int parameter to indicate how many times to rerun failing tests
+ */
+ public int getRerunFailingTestsCount()
+ {
+ return this.rerunFailingTestsCount;
+ }
+
private static List<File> createFiles( List suiteXmlFiles )
{
if ( suiteXmlFiles != null )
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
index 607c332..2c42c19 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/TestsToRun.java
@@ -138,4 +138,22 @@ public class TestsToRun implements Iterable<Class>
}
return result.toArray( new Class[result.size()] );
}
+
+ /**
+ * Get test class which matches className
+ *
+ * @param className string used to find the test class
+ * @return Class object with the matching name, null if could not find a class with the matching name
+ */
+ public Class getClassByName( String className )
+ {
+ for ( Class clazz : this )
+ {
+ if ( clazz.getName().equals( className ) )
+ {
+ return clazz;
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
index ec99c64..007a26c 100644
--- a/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
+++ b/surefire-api/src/test/java/org/apache/maven/surefire/suite/RunResultTest.java
@@ -58,6 +58,7 @@ public class RunResultTest
assertEquals( 3, simple.getErrors() );
assertEquals( 7, simple.getFailures() );
assertEquals( 4, simple.getSkipped() );
+ assertEquals( 2, simple.getFlakes() );
}
@@ -116,8 +117,8 @@ public class RunResultTest
private RunResult getSimpleAggregate()
{
- RunResult resultOne = new RunResult( 10, 1, 3, 2 );
- RunResult resultTwo = new RunResult( 10, 2, 4, 2 );
+ RunResult resultOne = new RunResult( 10, 1, 3, 2, 1 );
+ RunResult resultTwo = new RunResult( 10, 2, 4, 2, 1 );
return resultOne.aggregate( resultTwo );
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
----------------------------------------------------------------------
diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
index c7029c0..d569c00 100644
--- a/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
+++ b/surefire-api/src/test/java/org/apache/maven/surefire/util/TestsToRunTest.java
@@ -65,6 +65,14 @@ public class TestsToRunTest
assertEquals( 2, locatedClasses.length );
}
+ public void testGetClassByName()
+ {
+ TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) );
+ assertEquals( T1.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T1" ) );
+ assertEquals( T2.class, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T2" ) );
+ assertEquals( null, testsToRun.getClassByName( "org.apache.maven.surefire.util.TestsToRunTest$T3" ) );
+ }
+
class T1
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
index d53bfda..2f637f0 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -47,4 +47,5 @@ public interface BooterConstants
String PROVIDER_CONFIGURATION = "providerConfiguration";
String FORKTESTSET = "forkTestSet";
String FORKTESTSET_PREFER_TESTS_FROM_IN_STREAM = "preferTestsFromInStream";
+ String RERUN_FAILING_TESTS_COUNT = "rerunFailingTestsCount";
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
index a9028e0..fce7c42 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -78,6 +78,8 @@ public class BooterDeserializer
final String runOrder = properties.getProperty( RUN_ORDER );
final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
+ final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
+
DirectoryScannerParameters dirScannerParams =
new DirectoryScannerParameters( testClassesDirectory, includesList, excludesList, specificTestsList,
properties.getBooleanObjectProperty( FAILIFNOTESTS ), runOrder );
@@ -86,7 +88,8 @@ public class BooterDeserializer
TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
TestRequest testSuiteDefinition =
- new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod );
+ new TestRequest( testSuiteXmlFiles, sourceDirectory, requestedTest, requestedTestMethod,
+ rerunFailingTestsCount );
ReporterConfiguration reporterConfiguration =
new ReporterConfiguration( reportsDirectory, properties.getBooleanObjectProperty( ISTRIMSTACKTRACE ) );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
----------------------------------------------------------------------
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
index 8a40c63..7a09352 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/PropertiesWrapper.java
@@ -76,6 +76,11 @@ public class PropertiesWrapper
return Boolean.valueOf( properties.getProperty( propertyName ) );
}
+ public int getIntProperty( String propertyName )
+ {
+ return Integer.parseInt( properties.getProperty( propertyName ) );
+ }
+
public File getFileProperty( String key )
{
final String property = getProperty( key );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
new file mode 100644
index 0000000..47fb71a
--- /dev/null
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit4RerunFailingTestsIT.java
@@ -0,0 +1,278 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Test;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ */
+public class JUnit4RerunFailingTestsIT
+ extends SurefireJUnit4IntegrationTestCase
+{
+ private SurefireLauncher unpack()
+ {
+ return unpack( "/junit4-rerun-failing-tests" );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsWithOneRetry()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0,
+ 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+ verifyFailuresOneRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsTwoRetry()
+ throws Exception
+ {
+ // Four flakes, both tests have been re-run twice
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest()
+ .assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+ verifyFailuresTwoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunFailingErrorTestsFalse()
+ throws Exception
+ {
+ OutputValidator outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion(
+ "4.7" ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+ verifyFailuresNoRetryAllClasses( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestClass()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+ verifyFailuresOneRetryOneClass( outputValidator );
+ }
+
+ @Test
+ public void testRerunOneTestMethod()
+ throws Exception
+ {
+ OutputValidator outputValidator =
+ unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
+ 0, 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+
+ outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.7" ).maven().addGoal(
+ "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+ "-DuseUnlimitedThreads=true" ).addGoal(
+ "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
+ 0 );
+
+ verifyFailuresOneRetryOneMethod( outputValidator );
+ }
+
+ private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+ }
+
+ private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+ {
+ verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+ {
+ verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+ }
+
+ private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+ {
+ verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+ }
+
+ private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ outputValidator.verifyTextInLog( "Tests in error" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors,
+ int flakes )
+ {
+ outputValidator.verifyTextInLog( "Flaked tests" );
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+ outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+ outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+ outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ outputValidator.verifyTextInLog( "Failed tests" );
+ outputValidator.verifyTextInLog( "testFailingTestOne(junit4.FlakyFirstTimeTest)" );
+ outputValidator.verifyTextInLog( "ERROR" );
+ outputValidator.verifyTextInLog( "testErrorTestOne(junit4.FlakyFirstTimeTest)" );
+
+ verifyStatistics( outputValidator, run, failures, errors, flakes );
+ }
+
+ private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+ {
+ if ( flakes > 0 )
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: "
+ + flakes );
+ }
+ else
+ {
+ outputValidator.verifyTextInLog(
+ "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
index 7c8740a..c7afb92 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
@@ -40,6 +40,12 @@ public class HelperAssertions
assertTestSuiteResults( total, errors, failures, skipped, suite );
}
+ public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, File testDir )
+ {
+ IntegrationTestSuiteResults suite = parseTestResults( new File[]{ testDir } );
+ assertTestSuiteResults( total, errors, failures, skipped, flakes, suite );
+ }
+
/**
* assert that the reports in the specified testDir have the right summary statistics
*/
@@ -59,6 +65,13 @@ public class HelperAssertions
Assert.assertEquals( "wrong number of skipped", skipped, actualSuite.getSkipped() );
}
+ public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes,
+ IntegrationTestSuiteResults actualSuite )
+ {
+ assertTestSuiteResults(total, errors, failures, skipped, actualSuite);
+ Assert.assertEquals( "wrong number of flaky tests", flakes, actualSuite.getFlakes() );
+ }
+
public static IntegrationTestSuiteResults parseTestResults( File[] testDirs )
{
List<ReportTestSuite> reports = extractReports( testDirs );
@@ -77,15 +90,16 @@ public class HelperAssertions
public static IntegrationTestSuiteResults parseReportList( List<ReportTestSuite> reports )
{
Assert.assertTrue( "No reports!", reports.size() > 0 );
- int total = 0, errors = 0, failures = 0, skipped = 0;
+ int total = 0, errors = 0, failures = 0, skipped = 0, flakes = 0;
for ( ReportTestSuite report : reports )
{
total += report.getNumberOfTests();
errors += report.getNumberOfErrors();
failures += report.getNumberOfFailures();
skipped += report.getNumberOfSkipped();
+ flakes += report.getNumberOfFlakes();
}
- return new IntegrationTestSuiteResults( total, errors, failures, skipped );
+ return new IntegrationTestSuiteResults( total, errors, failures, skipped, flakes );
}
public static List<ReportTestSuite> extractReports( File[] testDirs )
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
index 8645027..f147281 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
@@ -22,7 +22,7 @@ package org.apache.maven.surefire.its.fixture;
public class IntegrationTestSuiteResults
{
- private int total, errors, failures, skipped;
+ private int total, errors, failures, skipped, flakes;
public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
{
@@ -32,6 +32,12 @@ public class IntegrationTestSuiteResults
this.skipped = skipped;
}
+ public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+ {
+ this(total, errors, failures, skipped);
+ this.flakes = flakes;
+ }
+
public int getTotal()
{
return total;
@@ -72,4 +78,14 @@ public class IntegrationTestSuiteResults
this.skipped = skipped;
}
+ public int getFlakes()
+ {
+ return flakes;
+ }
+
+ public void setFlakes( int flakes )
+ {
+ this.flakes = flakes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
index 2671879..14dfc47 100644
--- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
+++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
@@ -125,6 +125,12 @@ public class OutputValidator
return this;
}
+ public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+ {
+ HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, flakes, baseDir );
+ return this;
+ }
+
public OutputValidator assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
{
HelperAssertions.assertIntegrationTestSuiteResults( total, errors, failures, skipped, baseDir );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
new file mode 100644
index 0000000..7482811
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.maven.plugins.surefire</groupId>
+ <artifactId>junit4-rerun-failing-tests</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Test for rerun failing tests in JUnit 4</name>
+
+
+ <properties>
+ <junitVersion>4.4</junitVersion>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junitVersion}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.5</source>
+ <target>1.5</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
new file mode 100644
index 0000000..264462c
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/FlakyFirstTimeTest.java
@@ -0,0 +1,62 @@
+package junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class FlakyFirstTimeTest
+{
+ private static int failingCount = 0;
+
+ private static int errorCount = 0;
+
+
+ @Test
+ public void testFailingTestOne()
+ {
+ System.out.println( "Failing test" );
+ // This test will fail with only one retry, but will pass with two
+ if ( failingCount < 2 )
+ {
+ failingCount++;
+ Assert.fail( "Failing test" );
+ }
+ }
+
+ @Test
+ public void testErrorTestOne() throws Exception
+ {
+ System.out.println( "Error test" );
+ // This test will error out with only one retry, but will pass with two
+ if ( errorCount < 2 )
+ {
+ errorCount++;
+ throw new IllegalArgumentException("...");
+ }
+ }
+
+ @Test
+ public void testPassingTest() throws Exception
+ {
+ System.out.println( "Passing test" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
----------------------------------------------------------------------
diff --git a/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
new file mode 100644
index 0000000..7cb0b57
--- /dev/null
+++ b/surefire-integration-tests/src/test/resources/junit4-rerun-failing-tests/src/test/java/junit4/PassingTest.java
@@ -0,0 +1,39 @@
+package junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class PassingTest
+{
+ @Test
+ public void testPassingTestOne()
+ {
+ System.out.println( "Passing test one" );
+ }
+
+ @Test
+ public void testPassingTestTwo() throws Exception
+ {
+ System.out.println( "Passing test two" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
new file mode 100644
index 0000000..52ce708
--- /dev/null
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java
@@ -0,0 +1,109 @@
+package org.apache.maven.surefire.common.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.util.TestsToRun;
+import org.apache.maven.surefire.util.internal.StringUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.runner.notification.Failure;
+
+import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.isFailureInsideJUnitItself;
+
+/**
+ *
+ * Utility method used among all JUnit4 providers
+ *
+ * @author Qingzhou Luo
+ *
+ */
+public class JUnit4ProviderUtil
+{
+ /**
+ * Organize all the failures in previous run into a map between test classes and corresponding failing test methods
+ *
+ * @param allFailures all the failures in previous run
+ * @param testsToRun all the test classes
+ * @return a map between failing test classes and their corresponding failing test methods
+ */
+ public static Map<Class<?>, Set<String>> generateFailingTests( List<Failure> allFailures, TestsToRun testsToRun )
+ {
+ Map<Class<?>, Set<String>> testClassMethods = new HashMap<Class<?>, Set<String>>();
+
+ for ( Failure failure : allFailures )
+ {
+ if ( !isFailureInsideJUnitItself( failure ) )
+ {
+ // failure.getTestHeader() is in the format: method(class)
+ String[] testMethodClass = StringUtils.split( failure.getTestHeader(), "(" );
+ String testMethod = testMethodClass[0];
+ String testClass = StringUtils.split( testMethodClass[1], ")" )[0];
+ Class testClassObj = testsToRun.getClassByName( testClass );
+
+ if ( testClassObj == null )
+ {
+ continue;
+ }
+
+ Set<String> failingMethods = testClassMethods.get( testClassObj );
+ if ( failingMethods == null )
+ {
+ failingMethods = new HashSet<String>();
+ failingMethods.add( testMethod );
+ testClassMethods.put( testClassObj, failingMethods );
+ }
+ else
+ {
+ failingMethods.add( testMethod );
+ }
+ }
+ }
+ return testClassMethods;
+ }
+
+ /**
+ * Get the name of all test methods from a list of Failures
+ *
+ * @param allFailures the list of failures for a given test class
+ * @return the list of test method names
+ */
+ public static Set<String> generateFailingTests( List<Failure> allFailures )
+ {
+ Set<String> failingMethods = new HashSet<String>();
+
+ for ( Failure failure : allFailures )
+ {
+ if ( !isFailureInsideJUnitItself( failure ) )
+ {
+ // failure.getTestHeader() is in the format: method(class)
+ String testMethod = StringUtils.split( failure.getTestHeader(), "(" )[0];
+ failingMethods.add( testMethod );
+ }
+ }
+ return failingMethods;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
index c4b5b57..f21ab04 100644
--- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4RunListener.java
@@ -209,7 +209,7 @@ public class JUnit4RunListener
}
}
- private static boolean isFailureInsideJUnitItself( Failure failure )
+ public static boolean isFailureInsideJUnitItself( Failure failure )
{
return failure.getDescription().getDisplayName().equals( "Test mechanism" );
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
new file mode 100644
index 0000000..5cca0e8
--- /dev/null
+++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnitTestFailureListener.java
@@ -0,0 +1,35 @@
+package org.apache.maven.surefire.common.junit4;
+
+import org.junit.runner.notification.Failure;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test listener to record all the failures during one run
+ *
+ * @author Qingzhou Luo
+ */
+public class JUnitTestFailureListener
+ extends org.junit.runner.notification.RunListener
+{
+
+ List<Failure> allFailures = new ArrayList<Failure>();
+
+ @Override
+ public void testFailure( Failure failure )
+ throws java.lang.Exception
+ {
+ allFailures.add( failure );
+ }
+
+ public List<Failure> getAllFailures()
+ {
+ return allFailures;
+ }
+
+ public void reset()
+ {
+ allFailures.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
----------------------------------------------------------------------
diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
new file mode 100644
index 0000000..d21393a
--- /dev/null
+++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java
@@ -0,0 +1,85 @@
+package org.apache.maven.surefire.common.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.maven.surefire.util.TestsToRun;
+import org.junit.runner.Description;
+import org.junit.runner.notification.Failure;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests;
+
+/**
+ * @author Qingzhou Luo
+ */
+public class JUnit4ProviderUtilTest
+ extends TestCase
+{
+ public void testGenerateFailingTests()
+ {
+ TestsToRun testsToRun = new TestsToRun( Arrays.asList( new Class[]{ T1.class, T2.class } ) );
+ List<Failure> failures = new ArrayList<Failure>( );
+
+ Description test1Description = Description.createTestDescription( T1.class, "testOne" );
+ Description test2Description = Description.createTestDescription( T1.class, "testTwo" );
+ Description test3Description = Description.createTestDescription( T2.class, "testThree" );
+ Description test4Description = Description.createTestDescription( T2.class, "testFour" );
+ Description test5Description = Description.createSuiteDescription( "Test mechanism" );
+
+ failures.add( new Failure( test1Description, new AssertionError() ) );
+ failures.add( new Failure( test2Description, new AssertionError() ) );
+ failures.add( new Failure( test3Description, new RuntimeException() ) );
+ failures.add( new Failure( test4Description, new AssertionError() ) );
+ failures.add( new Failure( test5Description, new RuntimeException() ) );
+
+ Map<Class<?>, Set<String>> result = generateFailingTests( failures, testsToRun );
+
+ assertEquals( 2, result.size() );
+ Set<String> resultForT1 = result.get( T1.class );
+ Set<String> resultForT2 = result.get( T2.class );
+
+ Set<String> expectedResultForT1 = new HashSet<String>();
+ expectedResultForT1.add( "testOne" );
+ expectedResultForT1.add( "testTwo" );
+ Set<String> expectedResultForT2 = new HashSet<String>();
+ expectedResultForT2.add( "testThree" );
+ expectedResultForT2.add( "testFour" );
+
+ assertEquals( expectedResultForT1, resultForT1 );
+ assertEquals( expectedResultForT2, resultForT2 );
+ }
+
+ class T1
+ {
+
+ }
+
+ class T2
+ {
+
+ }
+}
\ No newline at end of file
[4/4] git commit: Adjust example page for rerun failing tests to be
valid for surefire and failsafe plugin
Posted by ag...@apache.org.
Adjust example page for rerun failing tests to be valid for surefire and failsafe plugin
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/2bdeeaf1
Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/2bdeeaf1
Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/2bdeeaf1
Branch: refs/heads/master
Commit: 2bdeeaf1ae6ff71b876fd881f807d1492399917b
Parents: fefaae7
Author: Andreas Gudian <ag...@apache.org>
Authored: Thu Jul 31 12:46:42 2014 +0200
Committer: Andreas Gudian <ag...@apache.org>
Committed: Thu Jul 31 12:46:42 2014 +0200
----------------------------------------------------------------------
.../src/site/apt/examples/rerun-failing-tests.apt.vm | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2bdeeaf1/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
----------------------------------------------------------------------
diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
index 42cc98c..607f875 100644
--- a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
+++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm
@@ -28,7 +28,6 @@
Rerun Failing Tests
-#{if}(${project.artifactId}=="maven-surefire-plugin")
During development, you may re-run failing tests because they are flaky.
To use this feature through Maven surefire, set the <<<rerunFailingTestsCount>>> property to be a value larger than 0.
Tests will be run until they pass or the number of reruns has been exhausted.
@@ -37,7 +36,7 @@ Rerun Failing Tests
+---+
-mvn -DrerunFailingTestsCount=2 test
+mvn -D${thisPlugin.toLowerCase()}.rerunFailingTestsCount=2 test
+---+
If <<<rerunFailingTestsCount>>> is set to a value smaller than or euqal to 0, then it will be ignored.
@@ -115,7 +114,7 @@ mvn -DrerunFailingTestsCount=2 test
2) The test fails in all of the re-runs:
<<<failure>>> and <<<error>>> elements will still be used in the generated xml report to include information
- for the first failing run, the same as without using <<<rerunFailingTests>>>. <<<rerunFailure>>> and <<<rerunError>>>
+ for the first failing run, the same as without using <<<rerunFailingTestsCount>>>. <<<rerunFailure>>> and <<<rerunError>>>
elements will be used in the generated xml report to include information of each <<subsequent>> failing re-runs.
<<<system-out>>> and <<<system-err>>> will also be used inside each <<<flakyFailure>>> or <<<flakyError>>> to include
information of System.out and System.err output. The original <<<system-out>>> and <<<system-err>>> elements will be
@@ -135,5 +134,3 @@ mvn -DrerunFailingTestsCount=2 test
In the xml report, the running time of a failing test with re-runs will be the running time of the
<<first failing run>>.
-
-#{end}
[3/4] git commit: Add rerunFailingTestsCount option for maven
surefire to rerun failing tests immediately after they fail.
Posted by ag...@apache.org.
Add rerunFailingTestsCount option for maven surefire to rerun failing tests immediately after they fail.
When rerunFailingTestsCount is set to a value larger k than 0, a failing test will get re-run
up to k times until it passes. If a test passes in any of its reruns,
the build will be marked as successful and the test will count as a
flake (or flaky test). If it fails all those k times then it will
still be marked as a failed test.
In the console output all the flaky tests will be count as "Flakes:
N". The generated test report XML file is augmented with additional
information, while still being compatible with existing consumers
(such as Jenkins). A flaky test will have <flakyFailure> or/and
<flakyError> under its <testcase> element, to store all the flaky
runs' information (such as output, stackTrace). So existing consumers
will still consider it as a passing test, while potential future
consumers can parse those flaky runs information. A failing test will
still have <failure> or <error> under <testcase>, but all the
subsequent re-run information will be stored under <rerunFailure> or
<rerunError>. So existing consumers will still be able to see it's a
failed test and parse its failure information, and potential future
consumers will be able to get all the flaky runs.
It is implemented by keeping a map between test full class name and a
map between all its test methods and the list of runs. It also takes
into account Fork and Parallel and have them covered by integration
tests.
Currently only supports JUnit4.x
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/fefaae7f
Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/fefaae7f
Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/fefaae7f
Branch: refs/heads/master
Commit: fefaae7f0534a59f52c046a64c96987e8561dd48
Parents: 1cdf49d
Author: Qingzhou Luo <qi...@google.com>
Authored: Tue Jun 24 13:44:44 2014 -0700
Committer: Qingzhou Luo <qi...@google.com>
Committed: Wed Jul 30 17:19:13 2014 -0700
----------------------------------------------------------------------
.../plugin/failsafe/IntegrationTestMojo.java | 12 +
maven-surefire-common/pom.xml | 6 +
.../plugin/surefire/AbstractSurefireMojo.java | 8 +-
.../maven/plugin/surefire/CommonReflector.java | 5 +-
.../surefire/StartupReportConfiguration.java | 20 +-
.../surefire/booterclient/BooterSerializer.java | 2 +
.../surefire/booterclient/ForkStarter.java | 26 +-
.../booterclient/output/ForkClient.java | 5 +
.../surefire/report/DefaultReporterFactory.java | 287 ++++++++++++++++--
.../plugin/surefire/report/ReportEntryType.java | 36 ++-
.../surefire/report/StatelessXmlReporter.java | 280 ++++++++++++++++--
.../plugin/surefire/report/TestMethodStats.java | 60 ++++
.../surefire/report/TestSetRunListener.java | 32 +-
.../plugin/surefire/report/TestSetStats.java | 10 +-
.../surefire/report/WrappedReportEntry.java | 5 +
.../maven/surefire/report/RunStatistics.java | 100 +++----
...erDeserializerProviderConfigurationTest.java | 5 +-
.../report/DefaultReporterFactoryTest.java | 206 +++++++++++++
.../report/StatelessXMLReporterTest.java | 172 -----------
.../report/StatelessXmlReporterTest.java | 290 +++++++++++++++++++
.../surefire/report/RunStatisticsTest.java | 87 +-----
.../maven/plugin/surefire/SurefirePlugin.java | 12 +
.../apt/examples/rerun-failing-tests.apt.vm | 139 +++++++++
maven-surefire-plugin/src/site/site.xml | 1 +
.../surefire/booter/SurefireReflector.java | 6 +-
.../apache/maven/surefire/suite/RunResult.java | 22 +-
.../maven/surefire/testset/TestRequest.java | 19 ++
.../apache/maven/surefire/util/TestsToRun.java | 18 ++
.../maven/surefire/suite/RunResultTest.java | 5 +-
.../maven/surefire/util/TestsToRunTest.java | 8 +
.../maven/surefire/booter/BooterConstants.java | 1 +
.../surefire/booter/BooterDeserializer.java | 5 +-
.../surefire/booter/PropertiesWrapper.java | 5 +
.../surefire/its/JUnit4RerunFailingTestsIT.java | 278 ++++++++++++++++++
.../surefire/its/fixture/HelperAssertions.java | 18 +-
.../fixture/IntegrationTestSuiteResults.java | 18 +-
.../surefire/its/fixture/OutputValidator.java | 6 +
.../junit4-rerun-failing-tests/pom.xml | 63 ++++
.../test/java/junit4/FlakyFirstTimeTest.java | 62 ++++
.../src/test/java/junit4/PassingTest.java | 39 +++
.../common/junit4/JUnit4ProviderUtil.java | 109 +++++++
.../common/junit4/JUnit4RunListener.java | 2 +-
.../common/junit4/JUnitTestFailureListener.java | 35 +++
.../common/junit4/JUnit4ProviderUtilTest.java | 85 ++++++
.../surefire/common/junit48/FilterFactory.java | 57 ++++
.../maven/surefire/junit4/JUnit4Provider.java | 61 +++-
.../surefire/junit4/JUnit4ProviderTest.java | 18 +-
.../surefire/junitcore/JUnitCoreProvider.java | 38 +++
.../junitcore/ConcurrentRunListenerTest.java | 5 +-
.../surefire/report/ReportTestSuite.java | 12 +
.../surefire/report/TestSuiteXmlParser.java | 4 +
51 files changed, 2375 insertions(+), 430 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
----------------------------------------------------------------------
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index 5439c35..1d47248 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -211,6 +211,18 @@ public class IntegrationTestMojo
@Parameter( property = "encoding", defaultValue = "${project.reporting.outputEncoding}" )
private String encoding;
+ /**
+ * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
+ * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
+ * However, all the failing attempts will be recorded.
+ */
+ @Parameter( property = "failsafe.rerunFailingTestsCount", defaultValue = "0" )
+ protected int rerunFailingTestsCount;
+
+ protected int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
protected void handleSummary( RunResult summary, Exception firstForkException )
throws MojoExecutionException, MojoFailureException
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/pom.xml
----------------------------------------------------------------------
diff --git a/maven-surefire-common/pom.xml b/maven-surefire-common/pom.xml
index ba36315..6386ca1 100644
--- a/maven-surefire-common/pom.xml
+++ b/maven-surefire-common/pom.xml
@@ -105,6 +105,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.4</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
index 730b0ad..6115cf3 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -429,7 +429,6 @@ public abstract class AbstractSurefireMojo
@Parameter( property = "threadCount" )
protected int threadCount;
-
/**
* Option to specify the number of VMs to fork in parallel in order to execute the tests.
* When terminated with "C", the number part is multiplied with the number of CPU cores. Floating point value are only accepted together with "C".
@@ -704,6 +703,8 @@ public abstract class AbstractSurefireMojo
protected abstract String getPluginName();
+ protected abstract int getRerunFailingTestsCount();
+
private SurefireDependencyResolver dependencyResolver;
public void execute()
@@ -1380,7 +1381,8 @@ public abstract class AbstractSurefireMojo
isTestNg ? new TestArtifactInfo( testNgArtifact.getVersion(), testNgArtifact.getClassifier() ) : null;
List<File> testXml = getSuiteXmlFiles() != null ? Arrays.asList( getSuiteXmlFiles() ) : null;
TestRequest testSuiteDefinition =
- new TestRequest( testXml, getTestSourceDirectory(), getTest(), getTestMethod() );
+ new TestRequest(testXml, getTestSourceDirectory(), getTest(), getTestMethod(),
+ getRerunFailingTestsCount());
final boolean failIfNoTests;
if ( isValidSuiteXmlFileConfig() && getTest() == null )
@@ -1492,7 +1494,7 @@ public abstract class AbstractSurefireMojo
return new StartupReportConfiguration( isUseFile(), isPrintSummary(), getReportFormat(),
isRedirectTestOutputToFile(), isDisableXmlReport(),
getReportsDirectory(), isTrimStackTrace(), getReportNameSuffix(),
- configChecksum, requiresRunHistory() );
+ configChecksum, requiresRunHistory(), getRerunFailingTestsCount() );
}
private boolean isSpecificTestSpecified()
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
index 6851539..ec48e42 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
@@ -66,13 +66,14 @@ public class CommonReflector
new Class[]{ boolean.class, boolean.class,
String.class, boolean.class, boolean.class,
File.class, boolean.class, String.class,
- String.class, boolean.class } );
+ String.class, boolean.class, int.class } );
//noinspection BooleanConstructorCall
final Object[] params = { reporterConfiguration.isUseFile(), reporterConfiguration.isPrintSummary(),
reporterConfiguration.getReportFormat(), reporterConfiguration.isRedirectTestOutputToFile(),
reporterConfiguration.isDisableXmlReport(), reporterConfiguration.getReportsDirectory(),
reporterConfiguration.isTrimStackTrace(), reporterConfiguration.getReportNameSuffix(),
- reporterConfiguration.getConfigurationHash(), reporterConfiguration.isRequiresRunHistory() };
+ reporterConfiguration.getConfigurationHash(), reporterConfiguration.isRequiresRunHistory(),
+ reporterConfiguration.getRerunFailingTestsCount() };
return ReflectionUtils.newInstance( constructor, params );
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
index f974ace..f44ba3a 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
@@ -64,6 +64,8 @@ public class StartupReportConfiguration
private final boolean trimStackTrace;
+ private final int rerunFailingTestsCount;
+
private final Properties testVmSystemProperties = new Properties();
public static final String BRIEF_REPORT_FORMAT = ConsoleReporter.BRIEF;
@@ -72,9 +74,9 @@ public class StartupReportConfiguration
public StartupReportConfiguration( boolean useFile, boolean printSummary, String reportFormat,
boolean redirectTestOutputToFile, boolean disableXmlReport,
- @Nonnull File reportsDirectory,
- boolean trimStackTrace, String reportNameSuffix,
- String configurationHash, boolean requiresRunHistory )
+ @Nonnull File reportsDirectory, boolean trimStackTrace, String reportNameSuffix,
+ String configurationHash, boolean requiresRunHistory,
+ int rerunFailingTestsCount )
{
this.useFile = useFile;
this.printSummary = printSummary;
@@ -88,20 +90,21 @@ public class StartupReportConfiguration
this.requiresRunHistory = requiresRunHistory;
this.originalSystemOut = System.out;
this.originalSystemErr = System.err;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
public static StartupReportConfiguration defaultValue()
{
File target = new File( "./target" );
return new StartupReportConfiguration( true, true, "PLAIN", false, false, target, false, null, "TESTHASH",
- false );
+ false, 0 );
}
public static StartupReportConfiguration defaultNoXml()
{
File target = new File( "./target" );
return new StartupReportConfiguration( true, true, "PLAIN", false, true, target, false, null, "TESTHASHxXML",
- false );
+ false, 0 );
}
public boolean isUseFile()
@@ -139,11 +142,16 @@ public class StartupReportConfiguration
return reportsDirectory;
}
+ public int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
public StatelessXmlReporter instantiateStatelessXmlReporter()
{
if ( !isDisableXmlReport() )
{
- return new StatelessXmlReporter( reportsDirectory, reportNameSuffix, trimStackTrace );
+ return new StatelessXmlReporter( reportsDirectory, reportNameSuffix, trimStackTrace,
+ rerunFailingTestsCount );
}
return null;
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
index 84371c8..d5c6181 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -95,6 +95,8 @@ class BooterSerializer
properties.addList( testSuiteDefinition.getSuiteXmlFiles(), BooterConstants.TEST_SUITE_XML_FILES );
properties.setNullableProperty( BooterConstants.REQUESTEDTEST, testSuiteDefinition.getRequestedTest() );
properties.setNullableProperty( BooterConstants.REQUESTEDTESTMETHOD, testSuiteDefinition.getRequestedTestMethod() );
+ properties.setNullableProperty( BooterConstants.RERUN_FAILING_TESTS_COUNT,
+ String.valueOf( testSuiteDefinition.getRerunFailingTestsCount() ) );
}
DirectoryScannerParameters directoryScannerParameters = booterConfiguration.getDirScannerParams();
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
index 8861e51..315c19d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
@@ -128,6 +128,8 @@ public class ForkStarter
private final DefaultReporterFactory defaultReporterFactory;
+ private final List<DefaultReporterFactory> defaultReporterFactoryList;
+
private static volatile int systemPropertiesFileCounter = 0;
public ForkStarter( ProviderConfiguration providerConfiguration, StartupConfiguration startupConfiguration,
@@ -141,6 +143,7 @@ public class ForkStarter
this.startupReportConfiguration = startupReportConfiguration;
this.log = log;
defaultReporterFactory = new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList = new ArrayList<DefaultReporterFactory>();
}
public RunResult run( SurefireProperties effectiveSystemProperties, DefaultScanResult scanResult )
@@ -153,8 +156,10 @@ public class ForkStarter
scanResult.writeTo( providerProperties );
if ( isForkOnce() )
{
+ DefaultReporterFactory forkedReporterFactory = new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
final ForkClient forkClient =
- new ForkClient( defaultReporterFactory, startupReportConfiguration.getTestVmSystemProperties() );
+ new ForkClient( forkedReporterFactory, startupReportConfiguration.getTestVmSystemProperties() );
result = fork( null, new PropertiesWrapper( providerProperties ), forkClient, effectiveSystemProperties,
null );
}
@@ -172,6 +177,7 @@ public class ForkStarter
}
finally
{
+ defaultReporterFactory.mergeFromOtherFactories(defaultReporterFactoryList);
defaultReporterFactory.close();
}
return result;
@@ -207,6 +213,7 @@ public class ForkStarter
messageQueue.add( clazz.getName() );
}
+
for ( int forkNum = 0; forkNum < forkCount && forkNum < suites.size(); forkNum++ )
{
Callable<RunResult> pf = new Callable<RunResult>()
@@ -217,7 +224,10 @@ public class ForkStarter
TestProvidingInputStream testProvidingInputStream =
new TestProvidingInputStream( messageQueue );
- ForkClient forkClient = new ForkClient( defaultReporterFactory,
+ DefaultReporterFactory forkedReporterFactory =
+ new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
+ ForkClient forkClient = new ForkClient( forkedReporterFactory,
startupReportConfiguration.getTestVmSystemProperties(),
testProvidingInputStream );
@@ -283,7 +293,10 @@ public class ForkStarter
public RunResult call()
throws Exception
{
- ForkClient forkClient = new ForkClient( defaultReporterFactory,
+ DefaultReporterFactory forkedReporterFactory =
+ new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
+ ForkClient forkClient = new ForkClient( forkedReporterFactory,
startupReportConfiguration.getTestVmSystemProperties() );
return fork( testSet, new PropertiesWrapper( providerConfiguration.getProviderProperties() ),
forkClient, effectiveSystemProperties, null );
@@ -451,11 +464,12 @@ public class ForkStarter
}
catch ( CommandLineTimeOutException e )
{
- runResult = RunResult.timeout( defaultReporterFactory.getGlobalRunStatistics().getRunResult() );
+ runResult = RunResult.timeout(
+ forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult() );
}
catch ( CommandLineException e )
{
- runResult = RunResult.failure( defaultReporterFactory.getGlobalRunStatistics().getRunResult(), e );
+ runResult = RunResult.failure( forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult(), e );
throw new SurefireBooterForkException( "Error while executing forked tests.", e.getCause() );
}
finally
@@ -468,7 +482,7 @@ public class ForkStarter
}
if ( runResult == null )
{
- runResult = defaultReporterFactory.getGlobalRunStatistics().getRunResult();
+ runResult = forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult();
}
if ( !runResult.isTimeout() )
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
index 0d8f7ff..75180c5 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
@@ -76,6 +76,11 @@ public class ForkClient
this.testProvidingInputStream = testProvidingInputStream;
}
+ public DefaultReporterFactory getDefaultReporterFactory()
+ {
+ return defaultReporterFactory;
+ }
+
public void consumeLine( String s )
{
try
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
index d02eddc..c2fff08 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -19,17 +19,22 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.maven.plugin.surefire.StartupReportConfiguration;
import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
import org.apache.maven.surefire.report.ReporterFactory;
import org.apache.maven.surefire.report.RunListener;
import org.apache.maven.surefire.report.RunStatistics;
+import org.apache.maven.surefire.report.StackTraceWriter;
import org.apache.maven.surefire.suite.RunResult;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
/**
* Provides reporting modules on the plugin side.
* <p/>
@@ -41,7 +46,7 @@ public class DefaultReporterFactory
implements ReporterFactory
{
- private final RunStatistics globalStats = new RunStatistics();
+ private RunStatistics globalStats = new RunStatistics();
private final StartupReportConfiguration reportConfiguration;
@@ -50,6 +55,15 @@ public class DefaultReporterFactory
private final List<TestSetRunListener> listeners =
Collections.synchronizedList( new ArrayList<TestSetRunListener>() );
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
+ private Map<String, List<TestMethodStats>> flakyTests = null;
+
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
+ private Map<String, List<TestMethodStats>> failedTests = null;
+
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
+ private Map<String, List<TestMethodStats>> errorTests = null;
+
public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
{
this.reportConfiguration = reportConfiguration;
@@ -62,6 +76,17 @@ public class DefaultReporterFactory
return createTestSetRunListener();
}
+ public void mergeFromOtherFactories( List<DefaultReporterFactory> factories )
+ {
+ for ( DefaultReporterFactory factory : factories )
+ {
+ for ( TestSetRunListener listener : factory.listeners )
+ {
+ listeners.add( listener );
+ }
+ }
+ }
+
public RunListener createTestSetRunListener()
{
TestSetRunListener testSetRunListener =
@@ -69,15 +94,21 @@ public class DefaultReporterFactory
reportConfiguration.instantiateFileReporter(),
reportConfiguration.instantiateStatelessXmlReporter(),
reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
- globalStats, reportConfiguration.isTrimStackTrace(),
+ reportConfiguration.isTrimStackTrace(),
ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
reportConfiguration.isBriefOrPlainFormat() );
listeners.add( testSetRunListener );
return testSetRunListener;
}
+ public void addListener( TestSetRunListener listener )
+ {
+ listeners.add( listener );
+ }
+
public RunResult close()
{
+ mergeTestHistoryResult();
runCompleted();
for ( TestSetRunListener listener : listeners )
{
@@ -109,35 +140,251 @@ public class DefaultReporterFactory
logger.info( "Results :" );
logger.info( "" );
}
- if ( globalStats.hadFailures() )
+ printTestFailures( logger, TestResultType.failure );
+ printTestFailures( logger, TestResultType.error );
+ printTestFailures( logger, TestResultType.flake );
+ logger.info( globalStats.getSummary() );
+ logger.info( "" );
+ }
+
+ public RunStatistics getGlobalRunStatistics()
+ {
+ mergeTestHistoryResult();
+ return globalStats;
+ }
+
+ public static DefaultReporterFactory defaultNoXml()
+ {
+ return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
+ }
+
+ /**
+ * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
+ * if it only has errors or failures, then count its result based on its first run
+ *
+ * @param reportEntryList the list of test run report type for a given test
+ * @return the type of test result
+ */
+ // Use default visibility for testing
+ static TestResultType getTestResultType( List<ReportEntryType> reportEntryList )
+ {
+ if ( reportEntryList == null || reportEntryList.size() == 0 )
+ {
+ return TestResultType.unknown;
+ }
+
+ boolean seenSuccess = false, seenFailure = false;
+ for ( ReportEntryType resultType : reportEntryList )
+ {
+ if ( resultType == ReportEntryType.success )
+ {
+ seenSuccess = true;
+ }
+ else if ( resultType == ReportEntryType.failure
+ || resultType == ReportEntryType.error )
+ {
+ seenFailure = true;
+ }
+ }
+
+ if ( seenSuccess && !seenFailure )
+ {
+ return TestResultType.success;
+ }
+
+ if ( seenSuccess && seenFailure )
+ {
+ return TestResultType.flake;
+ }
+
+ if ( !seenSuccess && seenFailure )
{
- logger.info( "Failed tests: " );
- for ( Object o : this.globalStats.getFailureSources() )
+ if ( reportEntryList.get( 0 ) == ReportEntryType.failure )
+ {
+ return TestResultType.failure;
+ }
+ else if ( reportEntryList.get( 0 ) == ReportEntryType.error )
{
- logger.info( " " + o );
+ return TestResultType.error;
}
+ else
+ {
+ // Reach here if the first one is skipped but later ones have failure, should be impossible
+ return TestResultType.skipped;
+ }
+ }
+ else
+ {
+ return TestResultType.skipped;
+ }
+ }
+
+ /**
+ * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and errorTests,
+ * indexed by test class and method name. Update globalStatistics based on the result of the merge.
+ */
+ void mergeTestHistoryResult()
+ {
+ globalStats = new RunStatistics();
+ flakyTests = new TreeMap<String, List<TestMethodStats>>();
+ failedTests = new TreeMap<String, List<TestMethodStats>>();
+ errorTests = new TreeMap<String, List<TestMethodStats>>();
+
+ Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
+ // Merge all the stats for tests from listeners
+ for ( TestSetRunListener listener : listeners )
+ {
+ List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
+ for ( TestMethodStats methodStats : testMethodStats )
+ {
+ List<TestMethodStats> currentMethodStats =
+ mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
+ if ( currentMethodStats == null )
+ {
+ currentMethodStats = new ArrayList<TestMethodStats>();
+ currentMethodStats.add( methodStats );
+ mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
+ }
+ else
+ {
+ currentMethodStats.add( methodStats );
+ }
+ }
+ }
+
+ // Update globalStatistics by iterating through mergedTestHistoryResult
+ int completedCount = 0, skipped = 0;
+
+ for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
+ {
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ String testClassMethodName = entry.getKey();
+ completedCount++;
+
+ List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
+ for (TestMethodStats methodStats : testMethodStats)
+ {
+ resultTypeList.add( methodStats.getResultType() );
+ }
+
+ TestResultType resultType = getTestResultType( resultTypeList );
+
+ switch ( resultType )
+ {
+ case success:
+ // If there are multiple successful runs of the same test, count all of them
+ int successCount = 0;
+ for (ReportEntryType type : resultTypeList) {
+ if (type == ReportEntryType.success) {
+ successCount++;
+ }
+ }
+ completedCount += successCount - 1;
+ break;
+ case skipped:
+ skipped++;
+ break;
+ case flake:
+ flakyTests.put( testClassMethodName, testMethodStats );
+ break;
+ case failure:
+ failedTests.put( testClassMethodName, testMethodStats );
+ break;
+ case error:
+ errorTests.put( testClassMethodName, testMethodStats );
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+
+ globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
+ }
+
+ /**
+ * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
+ * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
+ *
+ * @param logger the logger used to log information
+ * @param type the type of results to be printed, could be error, failure or flake
+ */
+ // Use default visibility for testing
+ void printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
+ {
+ Map<String, List<TestMethodStats>> testStats;
+ if ( type == TestResultType.failure )
+ {
+ testStats = failedTests;
+ }
+ else if ( type == TestResultType.error )
+ {
+ testStats = errorTests;
+ }
+ else if ( type == TestResultType.flake )
+ {
+ testStats = flakyTests;
+ }
+ else
+ {
logger.info( "" );
+ return;
+ }
+
+ if ( testStats.size() > 0 )
+ {
+ logger.info( type.getLogPrefix() );
}
- if ( globalStats.hadErrors() )
+
+ for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
{
- logger.info( "Tests in error: " );
- for ( Object o : this.globalStats.getErrorSources() )
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ if ( testMethodStats.size() == 1 )
{
- logger.info( " " + o );
+ // No rerun, follow the original output format
+ logger.info( " " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
+ continue;
+ }
+
+ logger.info( entry.getKey() );
+
+ for ( int i = 0; i < testMethodStats.size(); i++ )
+ {
+ StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
+ if ( failureStackTrace == null )
+ {
+ logger.info( " Run " + ( i + 1 ) + ": PASS" );
+ }
+ else
+ {
+ logger.info( " Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
+ }
}
logger.info( "" );
}
- logger.info( globalStats.getSummary() );
logger.info( "" );
}
- public RunStatistics getGlobalRunStatistics()
+ // Describe the result of a given test
+ static enum TestResultType
{
- return globalStats;
- }
- public static DefaultReporterFactory defaultNoXml()
- {
- return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
+ error( "Tests in error: " ),
+ failure( "Failed tests: " ),
+ flake( "Flaked tests: " ),
+ success( "Success: " ),
+ skipped( "Skipped: " ),
+ unknown( "Unknown: " );
+
+ private final String logPrefix;
+
+ private TestResultType( String logPrefix )
+ {
+ this.logPrefix = logPrefix;
+ }
+
+ public String getLogPrefix()
+ {
+ return logPrefix;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
index 562b123..e998c8e 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
@@ -21,9 +21,37 @@ package org.apache.maven.plugin.surefire.report;
public enum ReportEntryType
{
- error,
- failure,
- skipped,
- success
+ error( "error", "flakyError", "rerunError" ),
+ failure( "failure", "flakyFailure", "rerunFailure" ),
+ skipped( "skipped", "", "" ),
+ success( "", "", "" );
+
+ private final String xmlTag;
+
+ private final String flakyXmlTag;
+
+ private final String rerunXmlTag;
+
+ private ReportEntryType( String xmlTag, String flakyXmlTag, String rerunXmlTag )
+ {
+ this.xmlTag = xmlTag;
+ this.flakyXmlTag = flakyXmlTag;
+ this.rerunXmlTag = rerunXmlTag;
+ }
+
+ public String getXmlTag()
+ {
+ return xmlTag;
+ }
+
+ public String getFlakyXmlTag()
+ {
+ return flakyXmlTag;
+ }
+
+ public String getRerunXmlTag()
+ {
+ return rerunXmlTag;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
index a3e4455..d120c11 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
@@ -19,6 +19,12 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
+import org.apache.maven.shared.utils.io.IOUtil;
+import org.apache.maven.shared.utils.xml.XMLWriter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.ReporterException;
+import org.apache.maven.surefire.report.SafeThrowable;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
@@ -27,16 +33,17 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
-import org.apache.maven.shared.utils.io.IOUtil;
-import org.apache.maven.shared.utils.xml.XMLWriter;
-import org.apache.maven.surefire.report.ReportEntry;
-import org.apache.maven.surefire.report.ReporterException;
-import org.apache.maven.surefire.report.SafeThrowable;
-
+import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars;
/**
@@ -84,46 +91,152 @@ public class StatelessXmlReporter
private final boolean trimStackTrace;
+ private final int rerunFailingTestsCount;
+
+ // Map between test class name and a map between test method name
+ // and the list of runs for each test method
+ private Map<String, Map<String, List<WrappedReportEntry>>> testClassMethodRunHistoryMap =
+ Collections.synchronizedMap( new HashMap<String, Map<String, List<WrappedReportEntry>>>() );
- public StatelessXmlReporter( File reportsDirectory, String reportNameSuffix, boolean trimStackTrace )
+ public StatelessXmlReporter( File reportsDirectory, String reportNameSuffix, boolean trimStackTrace,
+ int rerunFailingTestsCount )
{
this.reportsDirectory = reportsDirectory;
this.reportNameSuffix = reportNameSuffix;
this.trimStackTrace = trimStackTrace;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStats testSetStats )
throws ReporterException
{
+ String testClassName = testSetReportEntry.getName();
+
+ Map<String, List<WrappedReportEntry>> methodRunHistoryMap = getAddMethodRunHistoryMap( testClassName );
+
+ // Update testClassMethodRunHistoryMap
+ for ( WrappedReportEntry methodEntry : testSetStats.getReportEntries() )
+ {
+ getAddMethodEntryList( methodRunHistoryMap, methodEntry );
+ }
FileOutputStream outputStream = getOutputStream( testSetReportEntry );
OutputStreamWriter fw = getWriter( outputStream );
try
{
-
org.apache.maven.shared.utils.xml.XMLWriter ppw =
new org.apache.maven.shared.utils.xml.PrettyPrintXMLWriter( fw );
ppw.setEncoding( ENCODING );
- createTestSuiteElement( ppw, testSetReportEntry, testSetStats, reportNameSuffix );
+ createTestSuiteElement( ppw, testSetReportEntry, testSetStats, reportNameSuffix,
+ testSetStats.elapsedTimeAsString( getRunTimeForAllTests( methodRunHistoryMap ) ) );
showProperties( ppw );
- for ( WrappedReportEntry entry : testSetStats.getReportEntries() )
+ // Iterate through all the test methods in the test class
+ for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() )
{
- if ( ReportEntryType.success.equals( entry.getReportEntryType() ) )
+ List<WrappedReportEntry> methodEntryList = entry.getValue();
+ if ( methodEntryList == null )
{
- startTestElement( ppw, entry, reportNameSuffix );
- ppw.endElement();
+ throw new IllegalStateException( "Get null test method run history" );
}
- else
+ if ( methodEntryList.size() == 0 )
{
- getTestProblems( fw, ppw, entry, trimStackTrace, reportNameSuffix, outputStream );
+ continue;
}
+ if ( rerunFailingTestsCount > 0 )
+ {
+ TestResultType resultType = getTestResultType( methodEntryList );
+ switch ( resultType )
+ {
+ case success:
+ for ( WrappedReportEntry methodEntry : methodEntryList )
+ {
+ if ( methodEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ startTestElement( ppw, methodEntry, reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ ppw.endElement();
+ }
+ }
+ break;
+ case error:
+ case failure:
+ // When rerunFailingTestsCount is set to larger than 0
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ boolean firstRun = true;
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( firstRun )
+ {
+ firstRun = false;
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getXmlTag(), false );
+ createOutErrElements( fw, ppw, singleRunEntry, outputStream );
+ }
+ else
+ {
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getRerunXmlTag(), true );
+ }
+ }
+ ppw.endElement();
+ break;
+ case flake:
+ String runtime = "";
+ // Get the run time of the first successful run
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ runtime = singleRunEntry.elapsedTimeAsString();
+ break;
+ }
+ }
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix, runtime );
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() != ReportEntryType.success )
+ {
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getFlakyXmlTag(), true );
+ }
+ }
+ ppw.endElement();
+
+ break;
+ case skipped:
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ getTestProblems( fw, ppw, methodEntryList.get( 0 ), trimStackTrace, outputStream,
+ methodEntryList.get( 0 ).getReportEntryType().getXmlTag(), false );
+ ppw.endElement();
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+ else
+ {
+ // rerunFailingTestsCount is smaller than 1, but for some reasons a test could be run
+ // for more than once
+ for ( WrappedReportEntry methodEntry : methodEntryList )
+ {
+ startTestElement( ppw, methodEntry, reportNameSuffix, methodEntry.elapsedTimeAsString() );
+ if ( methodEntry.getReportEntryType() != ReportEntryType.success )
+ {
+ getTestProblems( fw, ppw, methodEntry, trimStackTrace, outputStream,
+ methodEntry.getReportEntryType().getXmlTag(), false );
+ createOutErrElements( fw, ppw, methodEntry, outputStream );
+ }
+ ppw.endElement();
+ }
+ }
}
ppw.endElement(); // TestSuite
-
}
finally
{
@@ -131,11 +244,115 @@ public class StatelessXmlReporter
}
}
+ /**
+ * Clean testClassMethodRunHistoryMap
+ */
+ public void cleanTestHistoryMap() {
+ testClassMethodRunHistoryMap.clear();
+ }
+
+ /**
+ * Get the result of a test from a list of its runs in WrappedReportEntry
+ *
+ * @param methodEntryList the list of runs for a given test
+ * @return the TestResultType for the given test
+ */
+ private static TestResultType getTestResultType( List<WrappedReportEntry> methodEntryList )
+ {
+ List<ReportEntryType> testResultTypeList = new ArrayList<ReportEntryType>();
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ testResultTypeList.add( singleRunEntry.getReportEntryType() );
+ }
+
+ return DefaultReporterFactory.getTestResultType( testResultTypeList );
+ }
+
+ /**
+ * Get run time for the entire test suite (test class)
+ * For a successful/failed/error test, the run time is the first run
+ * For a flaky test, the run time is the first successful run's time
+ * The run time for the entire test class is the sum of all its test methods
+ *
+ *
+ * @param methodRunHistoryMap the input map between test method name and the list of all its runs
+ * in a given test class
+ * @return the run time for the entire test class
+ */
+ private static int getRunTimeForAllTests( Map<String, List<WrappedReportEntry>> methodRunHistoryMap )
+ {
+ int totalTimeForSuite = 0;
+ for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() )
+ {
+ List<WrappedReportEntry> methodEntryList = entry.getValue();
+ if ( methodEntryList == null )
+ {
+ throw new IllegalStateException( "Get null test method run history" );
+ }
+ if ( methodEntryList.size() == 0 )
+ {
+ continue;
+ }
+
+ TestResultType resultType = getTestResultType( methodEntryList );
+
+ switch ( resultType )
+ {
+ case success:
+ case error:
+ case failure:
+ // Get the first run's time for failure/error/success runs
+ totalTimeForSuite = totalTimeForSuite + methodEntryList.get( 0 ).getElapsed();
+ break;
+ case flake:
+ // Get the first successful run's time for flaky runs
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ totalTimeForSuite = totalTimeForSuite + singleRunEntry.getElapsed();
+ break;
+ }
+ }
+ break;
+ case skipped:
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+ return totalTimeForSuite;
+ }
+
private OutputStreamWriter getWriter( FileOutputStream fos )
{
return new OutputStreamWriter( fos, ENCODING_CS );
}
+
+ private void getAddMethodEntryList( Map<String, List<WrappedReportEntry>> methodRunHistoryMap,
+ WrappedReportEntry methodEntry )
+ {
+ List<WrappedReportEntry> methodEntryList = methodRunHistoryMap.get( methodEntry.getName() );
+ if ( methodEntryList == null )
+ {
+ methodEntryList = new ArrayList<WrappedReportEntry>();
+ methodRunHistoryMap.put( methodEntry.getName(), methodEntryList );
+ }
+ methodEntryList.add( methodEntry );
+ }
+
+ private Map<String, List<WrappedReportEntry>> getAddMethodRunHistoryMap( String testClassName )
+ {
+ Map<String, List<WrappedReportEntry>> methodRunHistoryMap = testClassMethodRunHistoryMap.get( testClassName );
+ if ( methodRunHistoryMap == null )
+ {
+ methodRunHistoryMap = Collections.synchronizedMap( new LinkedHashMap<String, List<WrappedReportEntry>>() );
+ testClassMethodRunHistoryMap.put( testClassName, methodRunHistoryMap );
+ }
+ return methodRunHistoryMap;
+ }
+
private FileOutputStream getOutputStream( WrappedReportEntry testSetReportEntry )
{
File reportFile = getReportFile( testSetReportEntry, reportsDirectory, reportNameSuffix );
@@ -173,7 +390,8 @@ public class StatelessXmlReporter
return reportFile;
}
- private static void startTestElement( XMLWriter ppw, WrappedReportEntry report, String reportNameSuffix )
+ private static void startTestElement( XMLWriter ppw, WrappedReportEntry report, String reportNameSuffix,
+ String timeAsString )
{
ppw.startElement( "testcase" );
ppw.addAttribute( "name", report.getReportName() );
@@ -192,11 +410,11 @@ public class StatelessXmlReporter
ppw.addAttribute( "classname", report.getSourceName() );
}
}
- ppw.addAttribute( "time", report.elapsedTimeAsString() );
+ ppw.addAttribute( "time", timeAsString );
}
private static void createTestSuiteElement( XMLWriter ppw, WrappedReportEntry report, TestSetStats testSetStats,
- String reportNameSuffix1 )
+ String reportNameSuffix1, String timeAsString )
{
ppw.startElement( "testsuite" );
@@ -207,7 +425,7 @@ public class StatelessXmlReporter
ppw.addAttribute( "group", report.getGroup() );
}
- ppw.addAttribute( "time", testSetStats.getElapsedForTestSet() );
+ ppw.addAttribute( "time", timeAsString );
ppw.addAttribute( "tests", String.valueOf( testSetStats.getCompletedCount() ) );
@@ -221,12 +439,10 @@ public class StatelessXmlReporter
private void getTestProblems( OutputStreamWriter outputStreamWriter, XMLWriter ppw, WrappedReportEntry report,
- boolean trimStackTrace, String reportNameSuffix, FileOutputStream fw )
+ boolean trimStackTrace, FileOutputStream fw,
+ String testErrorType, boolean createOutErrElementsInside )
{
-
- startTestElement( ppw, report, reportNameSuffix );
-
- ppw.startElement( report.getReportEntryType().name() );
+ ppw.startElement( testErrorType );
String stackTrace = report.getStackTrace( trimStackTrace );
@@ -259,15 +475,21 @@ public class StatelessXmlReporter
ppw.writeText( extraEscape( stackTrace, false ) );
}
+ if ( createOutErrElementsInside )
+ {
+ createOutErrElements( outputStreamWriter, ppw, report, fw );
+ }
+
ppw.endElement(); // entry type
+ }
+ // Create system-out and system-err elements
+ private void createOutErrElements( OutputStreamWriter outputStreamWriter, XMLWriter ppw, WrappedReportEntry report,
+ FileOutputStream fw )
+ {
EncodingOutputStream eos = new EncodingOutputStream( fw );
-
addOutputStreamElement( outputStreamWriter, fw, eos, ppw, report.getStdout(), "system-out" );
-
addOutputStreamElement( outputStreamWriter, fw, eos, ppw, report.getStdErr(), "system-err" );
-
- ppw.endElement(); // test element
}
private void addOutputStreamElement( OutputStreamWriter outputStreamWriter, OutputStream fw,
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
new file mode 100644
index 0000000..e202a48
--- /dev/null
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
@@ -0,0 +1,60 @@
+package org.apache.maven.plugin.surefire.report;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+/**
+ *
+ * Maintains per-thread test result state for a single test method.
+ *
+ * @author Qingzhou Luo
+ *
+ */
+public class TestMethodStats
+{
+ private final String testClassMethodName;
+
+ private final ReportEntryType resultType;
+
+ private final StackTraceWriter stackTraceWriter;
+
+ public TestMethodStats( String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter )
+ {
+ this.testClassMethodName = testClassMethodName;
+ this.resultType = resultType;
+ this.stackTraceWriter = stackTraceWriter;
+ }
+
+ public String getTestClassMethodName()
+ {
+ return testClassMethodName;
+ }
+
+ public ReportEntryType getResultType()
+ {
+ return resultType;
+ }
+
+ public StackTraceWriter getStackTraceWriter()
+ {
+ return stackTraceWriter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
index c100aae..ca2403d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
@@ -19,16 +19,14 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
-import org.apache.commons.io.output.DeferredFileOutputStream;
import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
import org.apache.maven.surefire.report.ConsoleLogger;
import org.apache.maven.surefire.report.ConsoleOutputReceiver;
import org.apache.maven.surefire.report.ReportEntry;
import org.apache.maven.surefire.report.RunListener;
-import org.apache.maven.surefire.report.RunStatistics;
-import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -40,10 +38,10 @@ import java.util.List;
public class TestSetRunListener
implements RunListener, ConsoleOutputReceiver, ConsoleLogger
{
- private final RunStatistics globalStatistics;
-
private final TestSetStats detailsForThis;
+ private List<TestMethodStats> testMethodStats;
+
private Utf8RecodingDeferredFileOutputStream testStdOut = initDeferred( "stdout" );
private Utf8RecodingDeferredFileOutputStream testStdErr = initDeferred( "stderr" );
@@ -68,7 +66,7 @@ public class TestSetRunListener
public TestSetRunListener( ConsoleReporter consoleReporter, FileReporter fileReporter,
StatelessXmlReporter simpleXMLReporter,
TestcycleConsoleOutputReceiver consoleOutputReceiver,
- StatisticsReporter statisticsReporter, RunStatistics globalStats, boolean trimStackTrace,
+ StatisticsReporter statisticsReporter, boolean trimStackTrace,
boolean isPlainFormat, boolean briefOrPlainFormat )
{
this.consoleReporter = consoleReporter;
@@ -78,7 +76,7 @@ public class TestSetRunListener
this.consoleOutputReceiver = consoleOutputReceiver;
this.briefOrPlainFormat = briefOrPlainFormat;
this.detailsForThis = new TestSetStats( trimStackTrace, isPlainFormat );
- this.globalStatistics = globalStats;
+ this.testMethodStats = new ArrayList<TestMethodStats>( );
}
public void info( String message )
@@ -154,7 +152,7 @@ public class TestSetRunListener
wrap.getStdout().free();
wrap.getStdErr().free();
- globalStatistics.add( detailsForThis );
+ addTestMethodStats();
detailsForThis.reset();
}
@@ -188,7 +186,6 @@ public class TestSetRunListener
{
statisticsReporter.testError( reportEntry );
}
- globalStatistics.addErrorSource( reportEntry.getStackTraceWriter() );
clearCapture();
}
@@ -200,7 +197,6 @@ public class TestSetRunListener
{
statisticsReporter.testFailed( reportEntry );
}
- globalStatistics.addFailureSource( reportEntry.getStackTraceWriter() );
clearCapture();
}
@@ -260,4 +256,20 @@ public class TestSetRunListener
consoleOutputReceiver.close();
}
}
+
+ public void addTestMethodStats()
+ {
+ for (WrappedReportEntry reportEntry : detailsForThis.getReportEntries())
+ {
+ TestMethodStats methodStats =
+ new TestMethodStats( reportEntry.getClassMethodName(), reportEntry.getReportEntryType(),
+ reportEntry.getStackTraceWriter() );
+ testMethodStats.add( methodStats );
+ }
+ }
+
+ public List<TestMethodStats> getTestMethodStats()
+ {
+ return testMethodStats;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
index b8078b6..a175b0a 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
@@ -154,17 +154,17 @@ public class TestSetStats
return skipped;
}
+ public String elapsedTimeAsString( long runTime )
+ {
+ return numberFormat.format( (double) runTime / MS_PER_SEC );
+ }
+
private static final String TEST_SET_COMPLETED_PREFIX = "Tests run: ";
private final NumberFormat numberFormat = NumberFormat.getInstance( Locale.ENGLISH );
private static final int MS_PER_SEC = 1000;
- String elapsedTimeAsString( long runTime )
- {
- return numberFormat.format( (double) runTime / MS_PER_SEC );
- }
-
public String getElapsedForTestSet()
{
return elapsedTimeAsString( elapsedForTestSet );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
index d38ae76..ef6961d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
@@ -87,6 +87,11 @@ public class WrappedReportEntry
return original.getName();
}
+ public String getClassMethodName()
+ {
+ return getSourceName() + "." + getName();
+ }
+
public String getGroup()
{
return original.getGroup();
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
index 139c3d7..926658f 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
@@ -19,27 +19,21 @@ package org.apache.maven.surefire.report;
* under the License.
*/
+import org.apache.maven.plugin.surefire.report.TestSetStats;
+import org.apache.maven.surefire.suite.RunResult;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import org.apache.maven.plugin.surefire.report.TestSetStats;
-import org.apache.maven.surefire.suite.RunResult;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
/**
* @author Kristian Rosenvold
*/
public class RunStatistics
{
- /**
- * Holds the source(s) that causes the error(s).
- */
- private final Sources errorSources = new Sources();
-
- /**
- * Holds the source(s) that causes the failure(s).
- */
- private final Sources failureSources = new Sources();
-
private int completedCount;
private int errors;
@@ -48,53 +42,45 @@ public class RunStatistics
private int skipped;
+ private int flakes;
- public void addErrorSource( StackTraceWriter stackTraceWriter )
+ public RunStatistics()
{
- if ( stackTraceWriter == null )
- {
- throw new IllegalArgumentException( "Cant be null" );
- }
- errorSources.addSource( stackTraceWriter );
}
- public void addFailureSource( StackTraceWriter stackTraceWriter )
+ public synchronized boolean hadFailures()
{
- if ( stackTraceWriter == null )
- {
- throw new IllegalArgumentException( "Cant be null" );
- }
- failureSources.addSource( stackTraceWriter );
+ return failures > 0;
}
- public Collection<String> getErrorSources()
+ public synchronized boolean hadErrors()
{
- return errorSources.getListOfSources();
+ return errors > 0;
}
- public Collection<String> getFailureSources()
+ public synchronized int getCompletedCount()
{
- return failureSources.getListOfSources();
+ return completedCount;
}
- public synchronized boolean hadFailures()
+ public synchronized int getSkipped()
{
- return failures > 0;
+ return skipped;
}
- public synchronized boolean hadErrors()
+ public synchronized int getFailures()
{
- return errors > 0;
+ return failures;
}
- public synchronized int getCompletedCount()
+ public synchronized int getErrors()
{
- return completedCount;
+ return errors;
}
- public synchronized int getSkipped()
+ public synchronized int getFlakes()
{
- return skipped;
+ return flakes;
}
public synchronized void add( TestSetStats testSetStats )
@@ -105,41 +91,29 @@ public class RunStatistics
this.skipped += testSetStats.getSkipped();
}
- public synchronized RunResult getRunResult()
+ public synchronized void set( int completedCount, int errors, int failures, int skipped, int flakes )
{
- return new RunResult( completedCount, errors, failures, skipped );
+ this.completedCount = completedCount;
+ this.errors = errors;
+ this.failures = failures;
+ this.skipped = skipped;
+ this.flakes = flakes;
}
- public synchronized String getSummary()
+ public synchronized RunResult getRunResult()
{
- return "Tests run: " + completedCount + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: "
- + skipped;
+ return new RunResult( completedCount, errors, failures, skipped, flakes );
}
-
- private static class Sources
+ public synchronized String getSummary()
{
- private final Collection<String> listOfSources = new ArrayList<String>();
-
- void addSource( String source )
- {
- synchronized ( listOfSources )
- {
- listOfSources.add( source );
- }
- }
-
- void addSource( StackTraceWriter stackTraceWriter )
- {
- addSource( stackTraceWriter.smartTrimmedStackTrace() );
- }
-
- Collection<String> getListOfSources()
+ String summary =
+ "Tests run: " + completedCount + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: "
+ + skipped;
+ if ( flakes > 0 )
{
- synchronized ( listOfSources )
- {
- return Collections.unmodifiableCollection( listOfSources );
- }
+ summary += ", Flakes: " + flakes;
}
+ return summary;
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
index c11ce59..f72d4cd 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
@@ -57,6 +57,8 @@ public class BooterDeserializerProviderConfigurationTest
private final String aUserRequestedTest = "aUserRequestedTest";
+ private final int rerunFailingTestsCount = 3;
+
private static ClassLoaderConfiguration getForkConfiguration()
{
return new ClassLoaderConfiguration( true, false );
@@ -124,6 +126,7 @@ public class BooterDeserializerProviderConfigurationTest
Assert.assertEquals( expected[1], suiteXmlFiles.get( 1 ) );
Assert.assertEquals( getTestSourceDirectory(), testSuiteDefinition.getTestSourceDirectory() );
Assert.assertEquals( aUserRequestedTest, testSuiteDefinition.getRequestedTest() );
+ Assert.assertEquals( rerunFailingTestsCount, testSuiteDefinition.getRerunFailingTestsCount() );
}
public void testTestForFork()
@@ -220,7 +223,7 @@ public class BooterDeserializerProviderConfigurationTest
String aUserRequestedTestMethod = "aUserRequestedTestMethod";
TestRequest testSuiteDefinition =
new TestRequest( getSuiteXmlFileStrings(), getTestSourceDirectory(), aUserRequestedTest,
- aUserRequestedTestMethod );
+ aUserRequestedTestMethod, rerunFailingTestsCount );
RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null );
return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration,
new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new Properties(),
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
new file mode 100644
index 0000000..0dc7865
--- /dev/null
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -0,0 +1,206 @@
+package org.apache.maven.plugin.surefire.report;
+
+import junit.framework.TestCase;
+import org.apache.maven.plugin.surefire.StartupReportConfiguration;
+import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.RunStatistics;
+import org.apache.maven.surefire.report.SafeThrowable;
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DefaultReporterFactoryTest
+ extends TestCase
+{
+
+ private final static String TEST_ONE = "testOne";
+
+ private final static String TEST_TWO = "testTwo";
+
+ private final static String TEST_THREE = "testThree";
+
+ private final static String TEST_FOUR = "testFour";
+
+ private final static String TEST_FIVE = "testFive";
+
+ private final static String ASSERTION_FAIL = "assertionFail";
+
+ private final static String ERROR = "error";
+
+ public void testMergeTestHistoryResult()
+ {
+ DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() );
+
+ // First run, four tests failed and one passed
+ List<TestMethodStats> firstRunStats = new ArrayList<TestMethodStats>();
+ firstRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ firstRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_FOUR, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_FIVE, ReportEntryType.success, null ) );
+
+ // Second run, two tests passed
+ List<TestMethodStats> secondRunStats = new ArrayList<TestMethodStats>();
+ secondRunStats.add(
+ new TestMethodStats( TEST_ONE, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ secondRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.success, null ) );
+ secondRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ secondRunStats.add( new TestMethodStats( TEST_FOUR, ReportEntryType.success, null ) );
+
+ // Third run, another test passed
+ List<TestMethodStats> thirdRunStats = new ArrayList<TestMethodStats>();
+ thirdRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.success, null ) );
+ thirdRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+
+ TestSetRunListener firstRunListener = mock( TestSetRunListener.class );
+ TestSetRunListener secondRunListener = mock( TestSetRunListener.class );
+ TestSetRunListener thirdRunListener = mock( TestSetRunListener.class );
+ when( firstRunListener.getTestMethodStats() ).thenReturn( firstRunStats );
+ when( secondRunListener.getTestMethodStats() ).thenReturn( secondRunStats );
+ when( thirdRunListener.getTestMethodStats() ).thenReturn( thirdRunStats );
+
+ factory.addListener( firstRunListener );
+ factory.addListener( secondRunListener );
+ factory.addListener( thirdRunListener );
+
+ factory.mergeTestHistoryResult();
+ RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
+
+ // Only TEST_THREE is a failing test, other three are flaky tests
+ assertEquals( 5, mergedStatistics.getCompletedCount() );
+ assertEquals( 0, mergedStatistics.getErrors() );
+ assertEquals( 1, mergedStatistics.getFailures() );
+ assertEquals( 3, mergedStatistics.getFlakes() );
+ assertEquals( 0, mergedStatistics.getSkipped() );
+
+ // Now test the result will be printed out correctly
+ DummyTestReporter reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.flake );
+ String[] expectedFlakeOutput =
+ { "Flaked tests: ", TEST_FOUR, " Run 1: " + ASSERTION_FAIL, " Run 2: PASS", "", TEST_ONE,
+ " Run 1: " + ERROR, " Run 2: " + ASSERTION_FAIL, " Run 3: PASS", "", TEST_TWO, " Run 1: " + ERROR,
+ " Run 2: PASS", "", "" };
+ assertEquals( Arrays.asList( expectedFlakeOutput ), reporter.getMessages() );
+
+ reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.failure );
+ String[] expectedFailureOutput =
+ { "Failed tests: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, "",
+ "" };
+ assertEquals( Arrays.asList( expectedFailureOutput ), reporter.getMessages() );
+
+ reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.error );
+ String[] expectedErrorOutput = { "" };
+ assertEquals( Arrays.asList( expectedErrorOutput ), reporter.getMessages() );
+ }
+
+ static class DummyTestReporter
+ extends DefaultDirectConsoleReporter
+ {
+
+ private final List<String> messages = new ArrayList<String>();
+
+ public DummyTestReporter()
+ {
+ super( System.out );
+ }
+
+ @Override
+ public void info( String msg )
+ {
+ messages.add( msg );
+ }
+
+ public List<String> getMessages()
+ {
+ return messages;
+ }
+ }
+
+ public void testGetTestResultType()
+ {
+ DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() );
+
+ List<ReportEntryType> emptyList = new ArrayList<ReportEntryType>();
+ assertEquals( unknown, factory.getTestResultType( emptyList ) );
+
+ List<ReportEntryType> successList = new ArrayList<ReportEntryType>();
+ successList.add( ReportEntryType.success );
+ successList.add( ReportEntryType.success );
+ assertEquals( success, factory.getTestResultType( successList ) );
+
+ List<ReportEntryType> failureErrorList = new ArrayList<ReportEntryType>();
+ failureErrorList.add( ReportEntryType.failure );
+ failureErrorList.add( ReportEntryType.error );
+ assertEquals( failure, factory.getTestResultType( failureErrorList ) );
+
+ List<ReportEntryType> errorFailureList = new ArrayList<ReportEntryType>();
+ errorFailureList.add( ReportEntryType.error );
+ errorFailureList.add( ReportEntryType.failure );
+ assertEquals( error, factory.getTestResultType( errorFailureList ) );
+
+ List<ReportEntryType> flakeList = new ArrayList<ReportEntryType>();
+ flakeList.add( ReportEntryType.success );
+ flakeList.add( ReportEntryType.failure );
+ assertEquals( flake, factory.getTestResultType( flakeList ) );
+
+ flakeList = new ArrayList<ReportEntryType>();
+ flakeList.add( ReportEntryType.error );
+ flakeList.add( ReportEntryType.success );
+ flakeList.add( ReportEntryType.failure );
+ assertEquals( flake, factory.getTestResultType( flakeList ) );
+
+ List<ReportEntryType> skippedList = new ArrayList<ReportEntryType>();
+ skippedList.add( ReportEntryType.skipped );
+ assertEquals( skipped, factory.getTestResultType( skippedList ) );
+ }
+
+ static class DummyStackTraceWriter
+ implements StackTraceWriter
+ {
+
+ private final String stackTrace;
+
+ public DummyStackTraceWriter( String stackTrace )
+ {
+ this.stackTrace = stackTrace;
+ }
+
+ @Override
+ public String writeTraceToString()
+ {
+ return "";
+ }
+
+ @Override
+ public String writeTrimmedTraceToString()
+ {
+ return "";
+ }
+
+ @Override
+ public String smartTrimmedStackTrace()
+ {
+ return stackTrace;
+ }
+
+ @Override
+ public SafeThrowable getThrowable()
+ {
+ return null;
+ }
+ }
+}
\ No newline at end of file