You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ag...@apache.org on 2014/07/31 12:47:43 UTC
[3/4] git commit: Add rerunFailingTestsCount option for maven
surefire to rerun failing tests immediately after they fail.
Add rerunFailingTestsCount option for maven surefire to rerun failing tests immediately after they fail.
When rerunFailingTestsCount is set to a value larger k than 0, a failing test will get re-run
up to k times until it passes. If a test passes in any of its reruns,
the build will be marked as successful and the test will count as a
flake (or flaky test). If it fails all those k times then it will
still be marked as a failed test.
In the console output all the flaky tests will be count as "Flakes:
N". The generated test report XML file is augmented with additional
information, while still being compatible with existing consumers
(such as Jenkins). A flaky test will have <flakyFailure> or/and
<flakyError> under its <testcase> element, to store all the flaky
runs' information (such as output, stackTrace). So existing consumers
will still consider it as a passing test, while potential future
consumers can parse those flaky runs information. A failing test will
still have <failure> or <error> under <testcase>, but all the
subsequent re-run information will be stored under <rerunFailure> or
<rerunError>. So existing consumers will still be able to see it's a
failed test and parse its failure information, and potential future
consumers will be able to get all the flaky runs.
It is implemented by keeping a map between test full class name and a
map between all its test methods and the list of runs. It also takes
into account Fork and Parallel and have them covered by integration
tests.
Currently only supports JUnit4.x
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/fefaae7f
Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/fefaae7f
Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/fefaae7f
Branch: refs/heads/master
Commit: fefaae7f0534a59f52c046a64c96987e8561dd48
Parents: 1cdf49d
Author: Qingzhou Luo <qi...@google.com>
Authored: Tue Jun 24 13:44:44 2014 -0700
Committer: Qingzhou Luo <qi...@google.com>
Committed: Wed Jul 30 17:19:13 2014 -0700
----------------------------------------------------------------------
.../plugin/failsafe/IntegrationTestMojo.java | 12 +
maven-surefire-common/pom.xml | 6 +
.../plugin/surefire/AbstractSurefireMojo.java | 8 +-
.../maven/plugin/surefire/CommonReflector.java | 5 +-
.../surefire/StartupReportConfiguration.java | 20 +-
.../surefire/booterclient/BooterSerializer.java | 2 +
.../surefire/booterclient/ForkStarter.java | 26 +-
.../booterclient/output/ForkClient.java | 5 +
.../surefire/report/DefaultReporterFactory.java | 287 ++++++++++++++++--
.../plugin/surefire/report/ReportEntryType.java | 36 ++-
.../surefire/report/StatelessXmlReporter.java | 280 ++++++++++++++++--
.../plugin/surefire/report/TestMethodStats.java | 60 ++++
.../surefire/report/TestSetRunListener.java | 32 +-
.../plugin/surefire/report/TestSetStats.java | 10 +-
.../surefire/report/WrappedReportEntry.java | 5 +
.../maven/surefire/report/RunStatistics.java | 100 +++----
...erDeserializerProviderConfigurationTest.java | 5 +-
.../report/DefaultReporterFactoryTest.java | 206 +++++++++++++
.../report/StatelessXMLReporterTest.java | 172 -----------
.../report/StatelessXmlReporterTest.java | 290 +++++++++++++++++++
.../surefire/report/RunStatisticsTest.java | 87 +-----
.../maven/plugin/surefire/SurefirePlugin.java | 12 +
.../apt/examples/rerun-failing-tests.apt.vm | 139 +++++++++
maven-surefire-plugin/src/site/site.xml | 1 +
.../surefire/booter/SurefireReflector.java | 6 +-
.../apache/maven/surefire/suite/RunResult.java | 22 +-
.../maven/surefire/testset/TestRequest.java | 19 ++
.../apache/maven/surefire/util/TestsToRun.java | 18 ++
.../maven/surefire/suite/RunResultTest.java | 5 +-
.../maven/surefire/util/TestsToRunTest.java | 8 +
.../maven/surefire/booter/BooterConstants.java | 1 +
.../surefire/booter/BooterDeserializer.java | 5 +-
.../surefire/booter/PropertiesWrapper.java | 5 +
.../surefire/its/JUnit4RerunFailingTestsIT.java | 278 ++++++++++++++++++
.../surefire/its/fixture/HelperAssertions.java | 18 +-
.../fixture/IntegrationTestSuiteResults.java | 18 +-
.../surefire/its/fixture/OutputValidator.java | 6 +
.../junit4-rerun-failing-tests/pom.xml | 63 ++++
.../test/java/junit4/FlakyFirstTimeTest.java | 62 ++++
.../src/test/java/junit4/PassingTest.java | 39 +++
.../common/junit4/JUnit4ProviderUtil.java | 109 +++++++
.../common/junit4/JUnit4RunListener.java | 2 +-
.../common/junit4/JUnitTestFailureListener.java | 35 +++
.../common/junit4/JUnit4ProviderUtilTest.java | 85 ++++++
.../surefire/common/junit48/FilterFactory.java | 57 ++++
.../maven/surefire/junit4/JUnit4Provider.java | 61 +++-
.../surefire/junit4/JUnit4ProviderTest.java | 18 +-
.../surefire/junitcore/JUnitCoreProvider.java | 38 +++
.../junitcore/ConcurrentRunListenerTest.java | 5 +-
.../surefire/report/ReportTestSuite.java | 12 +
.../surefire/report/TestSuiteXmlParser.java | 4 +
51 files changed, 2375 insertions(+), 430 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
----------------------------------------------------------------------
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index 5439c35..1d47248 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -211,6 +211,18 @@ public class IntegrationTestMojo
@Parameter( property = "encoding", defaultValue = "${project.reporting.outputEncoding}" )
private String encoding;
+ /**
+ * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
+ * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
+ * However, all the failing attempts will be recorded.
+ */
+ @Parameter( property = "failsafe.rerunFailingTestsCount", defaultValue = "0" )
+ protected int rerunFailingTestsCount;
+
+ protected int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
protected void handleSummary( RunResult summary, Exception firstForkException )
throws MojoExecutionException, MojoFailureException
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/pom.xml
----------------------------------------------------------------------
diff --git a/maven-surefire-common/pom.xml b/maven-surefire-common/pom.xml
index ba36315..6386ca1 100644
--- a/maven-surefire-common/pom.xml
+++ b/maven-surefire-common/pom.xml
@@ -105,6 +105,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.4</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
index 730b0ad..6115cf3 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -429,7 +429,6 @@ public abstract class AbstractSurefireMojo
@Parameter( property = "threadCount" )
protected int threadCount;
-
/**
* Option to specify the number of VMs to fork in parallel in order to execute the tests.
* When terminated with "C", the number part is multiplied with the number of CPU cores. Floating point value are only accepted together with "C".
@@ -704,6 +703,8 @@ public abstract class AbstractSurefireMojo
protected abstract String getPluginName();
+ protected abstract int getRerunFailingTestsCount();
+
private SurefireDependencyResolver dependencyResolver;
public void execute()
@@ -1380,7 +1381,8 @@ public abstract class AbstractSurefireMojo
isTestNg ? new TestArtifactInfo( testNgArtifact.getVersion(), testNgArtifact.getClassifier() ) : null;
List<File> testXml = getSuiteXmlFiles() != null ? Arrays.asList( getSuiteXmlFiles() ) : null;
TestRequest testSuiteDefinition =
- new TestRequest( testXml, getTestSourceDirectory(), getTest(), getTestMethod() );
+ new TestRequest(testXml, getTestSourceDirectory(), getTest(), getTestMethod(),
+ getRerunFailingTestsCount());
final boolean failIfNoTests;
if ( isValidSuiteXmlFileConfig() && getTest() == null )
@@ -1492,7 +1494,7 @@ public abstract class AbstractSurefireMojo
return new StartupReportConfiguration( isUseFile(), isPrintSummary(), getReportFormat(),
isRedirectTestOutputToFile(), isDisableXmlReport(),
getReportsDirectory(), isTrimStackTrace(), getReportNameSuffix(),
- configChecksum, requiresRunHistory() );
+ configChecksum, requiresRunHistory(), getRerunFailingTestsCount() );
}
private boolean isSpecificTestSpecified()
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
index 6851539..ec48e42 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java
@@ -66,13 +66,14 @@ public class CommonReflector
new Class[]{ boolean.class, boolean.class,
String.class, boolean.class, boolean.class,
File.class, boolean.class, String.class,
- String.class, boolean.class } );
+ String.class, boolean.class, int.class } );
//noinspection BooleanConstructorCall
final Object[] params = { reporterConfiguration.isUseFile(), reporterConfiguration.isPrintSummary(),
reporterConfiguration.getReportFormat(), reporterConfiguration.isRedirectTestOutputToFile(),
reporterConfiguration.isDisableXmlReport(), reporterConfiguration.getReportsDirectory(),
reporterConfiguration.isTrimStackTrace(), reporterConfiguration.getReportNameSuffix(),
- reporterConfiguration.getConfigurationHash(), reporterConfiguration.isRequiresRunHistory() };
+ reporterConfiguration.getConfigurationHash(), reporterConfiguration.isRequiresRunHistory(),
+ reporterConfiguration.getRerunFailingTestsCount() };
return ReflectionUtils.newInstance( constructor, params );
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
index f974ace..f44ba3a 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java
@@ -64,6 +64,8 @@ public class StartupReportConfiguration
private final boolean trimStackTrace;
+ private final int rerunFailingTestsCount;
+
private final Properties testVmSystemProperties = new Properties();
public static final String BRIEF_REPORT_FORMAT = ConsoleReporter.BRIEF;
@@ -72,9 +74,9 @@ public class StartupReportConfiguration
public StartupReportConfiguration( boolean useFile, boolean printSummary, String reportFormat,
boolean redirectTestOutputToFile, boolean disableXmlReport,
- @Nonnull File reportsDirectory,
- boolean trimStackTrace, String reportNameSuffix,
- String configurationHash, boolean requiresRunHistory )
+ @Nonnull File reportsDirectory, boolean trimStackTrace, String reportNameSuffix,
+ String configurationHash, boolean requiresRunHistory,
+ int rerunFailingTestsCount )
{
this.useFile = useFile;
this.printSummary = printSummary;
@@ -88,20 +90,21 @@ public class StartupReportConfiguration
this.requiresRunHistory = requiresRunHistory;
this.originalSystemOut = System.out;
this.originalSystemErr = System.err;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
public static StartupReportConfiguration defaultValue()
{
File target = new File( "./target" );
return new StartupReportConfiguration( true, true, "PLAIN", false, false, target, false, null, "TESTHASH",
- false );
+ false, 0 );
}
public static StartupReportConfiguration defaultNoXml()
{
File target = new File( "./target" );
return new StartupReportConfiguration( true, true, "PLAIN", false, true, target, false, null, "TESTHASHxXML",
- false );
+ false, 0 );
}
public boolean isUseFile()
@@ -139,11 +142,16 @@ public class StartupReportConfiguration
return reportsDirectory;
}
+ public int getRerunFailingTestsCount() {
+ return rerunFailingTestsCount;
+ }
+
public StatelessXmlReporter instantiateStatelessXmlReporter()
{
if ( !isDisableXmlReport() )
{
- return new StatelessXmlReporter( reportsDirectory, reportNameSuffix, trimStackTrace );
+ return new StatelessXmlReporter( reportsDirectory, reportNameSuffix, trimStackTrace,
+ rerunFailingTestsCount );
}
return null;
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
index 84371c8..d5c6181 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -95,6 +95,8 @@ class BooterSerializer
properties.addList( testSuiteDefinition.getSuiteXmlFiles(), BooterConstants.TEST_SUITE_XML_FILES );
properties.setNullableProperty( BooterConstants.REQUESTEDTEST, testSuiteDefinition.getRequestedTest() );
properties.setNullableProperty( BooterConstants.REQUESTEDTESTMETHOD, testSuiteDefinition.getRequestedTestMethod() );
+ properties.setNullableProperty( BooterConstants.RERUN_FAILING_TESTS_COUNT,
+ String.valueOf( testSuiteDefinition.getRerunFailingTestsCount() ) );
}
DirectoryScannerParameters directoryScannerParameters = booterConfiguration.getDirScannerParams();
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
index 8861e51..315c19d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
@@ -128,6 +128,8 @@ public class ForkStarter
private final DefaultReporterFactory defaultReporterFactory;
+ private final List<DefaultReporterFactory> defaultReporterFactoryList;
+
private static volatile int systemPropertiesFileCounter = 0;
public ForkStarter( ProviderConfiguration providerConfiguration, StartupConfiguration startupConfiguration,
@@ -141,6 +143,7 @@ public class ForkStarter
this.startupReportConfiguration = startupReportConfiguration;
this.log = log;
defaultReporterFactory = new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList = new ArrayList<DefaultReporterFactory>();
}
public RunResult run( SurefireProperties effectiveSystemProperties, DefaultScanResult scanResult )
@@ -153,8 +156,10 @@ public class ForkStarter
scanResult.writeTo( providerProperties );
if ( isForkOnce() )
{
+ DefaultReporterFactory forkedReporterFactory = new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
final ForkClient forkClient =
- new ForkClient( defaultReporterFactory, startupReportConfiguration.getTestVmSystemProperties() );
+ new ForkClient( forkedReporterFactory, startupReportConfiguration.getTestVmSystemProperties() );
result = fork( null, new PropertiesWrapper( providerProperties ), forkClient, effectiveSystemProperties,
null );
}
@@ -172,6 +177,7 @@ public class ForkStarter
}
finally
{
+ defaultReporterFactory.mergeFromOtherFactories(defaultReporterFactoryList);
defaultReporterFactory.close();
}
return result;
@@ -207,6 +213,7 @@ public class ForkStarter
messageQueue.add( clazz.getName() );
}
+
for ( int forkNum = 0; forkNum < forkCount && forkNum < suites.size(); forkNum++ )
{
Callable<RunResult> pf = new Callable<RunResult>()
@@ -217,7 +224,10 @@ public class ForkStarter
TestProvidingInputStream testProvidingInputStream =
new TestProvidingInputStream( messageQueue );
- ForkClient forkClient = new ForkClient( defaultReporterFactory,
+ DefaultReporterFactory forkedReporterFactory =
+ new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
+ ForkClient forkClient = new ForkClient( forkedReporterFactory,
startupReportConfiguration.getTestVmSystemProperties(),
testProvidingInputStream );
@@ -283,7 +293,10 @@ public class ForkStarter
public RunResult call()
throws Exception
{
- ForkClient forkClient = new ForkClient( defaultReporterFactory,
+ DefaultReporterFactory forkedReporterFactory =
+ new DefaultReporterFactory( startupReportConfiguration );
+ defaultReporterFactoryList.add( forkedReporterFactory );
+ ForkClient forkClient = new ForkClient( forkedReporterFactory,
startupReportConfiguration.getTestVmSystemProperties() );
return fork( testSet, new PropertiesWrapper( providerConfiguration.getProviderProperties() ),
forkClient, effectiveSystemProperties, null );
@@ -451,11 +464,12 @@ public class ForkStarter
}
catch ( CommandLineTimeOutException e )
{
- runResult = RunResult.timeout( defaultReporterFactory.getGlobalRunStatistics().getRunResult() );
+ runResult = RunResult.timeout(
+ forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult() );
}
catch ( CommandLineException e )
{
- runResult = RunResult.failure( defaultReporterFactory.getGlobalRunStatistics().getRunResult(), e );
+ runResult = RunResult.failure( forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult(), e );
throw new SurefireBooterForkException( "Error while executing forked tests.", e.getCause() );
}
finally
@@ -468,7 +482,7 @@ public class ForkStarter
}
if ( runResult == null )
{
- runResult = defaultReporterFactory.getGlobalRunStatistics().getRunResult();
+ runResult = forkClient.getDefaultReporterFactory().getGlobalRunStatistics().getRunResult();
}
if ( !runResult.isTimeout() )
{
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
index 0d8f7ff..75180c5 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
@@ -76,6 +76,11 @@ public class ForkClient
this.testProvidingInputStream = testProvidingInputStream;
}
+ public DefaultReporterFactory getDefaultReporterFactory()
+ {
+ return defaultReporterFactory;
+ }
+
public void consumeLine( String s )
{
try
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
index d02eddc..c2fff08 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -19,17 +19,22 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.maven.plugin.surefire.StartupReportConfiguration;
import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
import org.apache.maven.surefire.report.ReporterFactory;
import org.apache.maven.surefire.report.RunListener;
import org.apache.maven.surefire.report.RunStatistics;
+import org.apache.maven.surefire.report.StackTraceWriter;
import org.apache.maven.surefire.suite.RunResult;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
/**
* Provides reporting modules on the plugin side.
* <p/>
@@ -41,7 +46,7 @@ public class DefaultReporterFactory
implements ReporterFactory
{
- private final RunStatistics globalStats = new RunStatistics();
+ private RunStatistics globalStats = new RunStatistics();
private final StartupReportConfiguration reportConfiguration;
@@ -50,6 +55,15 @@ public class DefaultReporterFactory
private final List<TestSetRunListener> listeners =
Collections.synchronizedList( new ArrayList<TestSetRunListener>() );
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for flaky tests
+ private Map<String, List<TestMethodStats>> flakyTests = null;
+
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for failed tests
+ private Map<String, List<TestMethodStats>> failedTests = null;
+
+ // from "<testclass>.<testmethod>" -> statistics about all the runs for error tests
+ private Map<String, List<TestMethodStats>> errorTests = null;
+
public DefaultReporterFactory( StartupReportConfiguration reportConfiguration )
{
this.reportConfiguration = reportConfiguration;
@@ -62,6 +76,17 @@ public class DefaultReporterFactory
return createTestSetRunListener();
}
+ public void mergeFromOtherFactories( List<DefaultReporterFactory> factories )
+ {
+ for ( DefaultReporterFactory factory : factories )
+ {
+ for ( TestSetRunListener listener : factory.listeners )
+ {
+ listeners.add( listener );
+ }
+ }
+ }
+
public RunListener createTestSetRunListener()
{
TestSetRunListener testSetRunListener =
@@ -69,15 +94,21 @@ public class DefaultReporterFactory
reportConfiguration.instantiateFileReporter(),
reportConfiguration.instantiateStatelessXmlReporter(),
reportConfiguration.instantiateConsoleOutputFileReporter(), statisticsReporter,
- globalStats, reportConfiguration.isTrimStackTrace(),
+ reportConfiguration.isTrimStackTrace(),
ConsoleReporter.PLAIN.equals( reportConfiguration.getReportFormat() ),
reportConfiguration.isBriefOrPlainFormat() );
listeners.add( testSetRunListener );
return testSetRunListener;
}
+ public void addListener( TestSetRunListener listener )
+ {
+ listeners.add( listener );
+ }
+
public RunResult close()
{
+ mergeTestHistoryResult();
runCompleted();
for ( TestSetRunListener listener : listeners )
{
@@ -109,35 +140,251 @@ public class DefaultReporterFactory
logger.info( "Results :" );
logger.info( "" );
}
- if ( globalStats.hadFailures() )
+ printTestFailures( logger, TestResultType.failure );
+ printTestFailures( logger, TestResultType.error );
+ printTestFailures( logger, TestResultType.flake );
+ logger.info( globalStats.getSummary() );
+ logger.info( "" );
+ }
+
+ public RunStatistics getGlobalRunStatistics()
+ {
+ mergeTestHistoryResult();
+ return globalStats;
+ }
+
+ public static DefaultReporterFactory defaultNoXml()
+ {
+ return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
+ }
+
+ /**
+ * Get the result of a test based on all its runs. If it has success and failures/errors, then it is a flake;
+ * if it only has errors or failures, then count its result based on its first run
+ *
+ * @param reportEntryList the list of test run report type for a given test
+ * @return the type of test result
+ */
+ // Use default visibility for testing
+ static TestResultType getTestResultType( List<ReportEntryType> reportEntryList )
+ {
+ if ( reportEntryList == null || reportEntryList.size() == 0 )
+ {
+ return TestResultType.unknown;
+ }
+
+ boolean seenSuccess = false, seenFailure = false;
+ for ( ReportEntryType resultType : reportEntryList )
+ {
+ if ( resultType == ReportEntryType.success )
+ {
+ seenSuccess = true;
+ }
+ else if ( resultType == ReportEntryType.failure
+ || resultType == ReportEntryType.error )
+ {
+ seenFailure = true;
+ }
+ }
+
+ if ( seenSuccess && !seenFailure )
+ {
+ return TestResultType.success;
+ }
+
+ if ( seenSuccess && seenFailure )
+ {
+ return TestResultType.flake;
+ }
+
+ if ( !seenSuccess && seenFailure )
{
- logger.info( "Failed tests: " );
- for ( Object o : this.globalStats.getFailureSources() )
+ if ( reportEntryList.get( 0 ) == ReportEntryType.failure )
+ {
+ return TestResultType.failure;
+ }
+ else if ( reportEntryList.get( 0 ) == ReportEntryType.error )
{
- logger.info( " " + o );
+ return TestResultType.error;
}
+ else
+ {
+ // Reach here if the first one is skipped but later ones have failure, should be impossible
+ return TestResultType.skipped;
+ }
+ }
+ else
+ {
+ return TestResultType.skipped;
+ }
+ }
+
+ /**
+ * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and errorTests,
+ * indexed by test class and method name. Update globalStatistics based on the result of the merge.
+ */
+ void mergeTestHistoryResult()
+ {
+ globalStats = new RunStatistics();
+ flakyTests = new TreeMap<String, List<TestMethodStats>>();
+ failedTests = new TreeMap<String, List<TestMethodStats>>();
+ errorTests = new TreeMap<String, List<TestMethodStats>>();
+
+ Map<String, List<TestMethodStats>> mergedTestHistoryResult = new HashMap<String, List<TestMethodStats>>();
+ // Merge all the stats for tests from listeners
+ for ( TestSetRunListener listener : listeners )
+ {
+ List<TestMethodStats> testMethodStats = listener.getTestMethodStats();
+ for ( TestMethodStats methodStats : testMethodStats )
+ {
+ List<TestMethodStats> currentMethodStats =
+ mergedTestHistoryResult.get( methodStats.getTestClassMethodName() );
+ if ( currentMethodStats == null )
+ {
+ currentMethodStats = new ArrayList<TestMethodStats>();
+ currentMethodStats.add( methodStats );
+ mergedTestHistoryResult.put( methodStats.getTestClassMethodName(), currentMethodStats );
+ }
+ else
+ {
+ currentMethodStats.add( methodStats );
+ }
+ }
+ }
+
+ // Update globalStatistics by iterating through mergedTestHistoryResult
+ int completedCount = 0, skipped = 0;
+
+ for ( Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet() )
+ {
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ String testClassMethodName = entry.getKey();
+ completedCount++;
+
+ List<ReportEntryType> resultTypeList = new ArrayList<ReportEntryType>();
+ for (TestMethodStats methodStats : testMethodStats)
+ {
+ resultTypeList.add( methodStats.getResultType() );
+ }
+
+ TestResultType resultType = getTestResultType( resultTypeList );
+
+ switch ( resultType )
+ {
+ case success:
+ // If there are multiple successful runs of the same test, count all of them
+ int successCount = 0;
+ for (ReportEntryType type : resultTypeList) {
+ if (type == ReportEntryType.success) {
+ successCount++;
+ }
+ }
+ completedCount += successCount - 1;
+ break;
+ case skipped:
+ skipped++;
+ break;
+ case flake:
+ flakyTests.put( testClassMethodName, testMethodStats );
+ break;
+ case failure:
+ failedTests.put( testClassMethodName, testMethodStats );
+ break;
+ case error:
+ errorTests.put( testClassMethodName, testMethodStats );
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+
+ globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() );
+ }
+
+ /**
+ * Print failed tests and flaked tests. A test is considered as a failed test if it failed/got an error with
+ * all the runs. If a test passes in ever of the reruns, it will be count as a flaked test
+ *
+ * @param logger the logger used to log information
+ * @param type the type of results to be printed, could be error, failure or flake
+ */
+ // Use default visibility for testing
+ void printTestFailures( DefaultDirectConsoleReporter logger, TestResultType type )
+ {
+ Map<String, List<TestMethodStats>> testStats;
+ if ( type == TestResultType.failure )
+ {
+ testStats = failedTests;
+ }
+ else if ( type == TestResultType.error )
+ {
+ testStats = errorTests;
+ }
+ else if ( type == TestResultType.flake )
+ {
+ testStats = flakyTests;
+ }
+ else
+ {
logger.info( "" );
+ return;
+ }
+
+ if ( testStats.size() > 0 )
+ {
+ logger.info( type.getLogPrefix() );
}
- if ( globalStats.hadErrors() )
+
+ for ( Map.Entry<String, List<TestMethodStats>> entry : testStats.entrySet() )
{
- logger.info( "Tests in error: " );
- for ( Object o : this.globalStats.getErrorSources() )
+ List<TestMethodStats> testMethodStats = entry.getValue();
+ if ( testMethodStats.size() == 1 )
{
- logger.info( " " + o );
+ // No rerun, follow the original output format
+ logger.info( " " + testMethodStats.get( 0 ).getStackTraceWriter().smartTrimmedStackTrace() );
+ continue;
+ }
+
+ logger.info( entry.getKey() );
+
+ for ( int i = 0; i < testMethodStats.size(); i++ )
+ {
+ StackTraceWriter failureStackTrace = testMethodStats.get( i ).getStackTraceWriter();
+ if ( failureStackTrace == null )
+ {
+ logger.info( " Run " + ( i + 1 ) + ": PASS" );
+ }
+ else
+ {
+ logger.info( " Run " + ( i + 1 ) + ": " + failureStackTrace.smartTrimmedStackTrace() );
+ }
}
logger.info( "" );
}
- logger.info( globalStats.getSummary() );
logger.info( "" );
}
- public RunStatistics getGlobalRunStatistics()
+ // Describe the result of a given test
+ static enum TestResultType
{
- return globalStats;
- }
- public static DefaultReporterFactory defaultNoXml()
- {
- return new DefaultReporterFactory( StartupReportConfiguration.defaultNoXml() );
+ error( "Tests in error: " ),
+ failure( "Failed tests: " ),
+ flake( "Flaked tests: " ),
+ success( "Success: " ),
+ skipped( "Skipped: " ),
+ unknown( "Unknown: " );
+
+ private final String logPrefix;
+
+ private TestResultType( String logPrefix )
+ {
+ this.logPrefix = logPrefix;
+ }
+
+ public String getLogPrefix()
+ {
+ return logPrefix;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
index 562b123..e998c8e 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/ReportEntryType.java
@@ -21,9 +21,37 @@ package org.apache.maven.plugin.surefire.report;
public enum ReportEntryType
{
- error,
- failure,
- skipped,
- success
+ error( "error", "flakyError", "rerunError" ),
+ failure( "failure", "flakyFailure", "rerunFailure" ),
+ skipped( "skipped", "", "" ),
+ success( "", "", "" );
+
+ private final String xmlTag;
+
+ private final String flakyXmlTag;
+
+ private final String rerunXmlTag;
+
+ private ReportEntryType( String xmlTag, String flakyXmlTag, String rerunXmlTag )
+ {
+ this.xmlTag = xmlTag;
+ this.flakyXmlTag = flakyXmlTag;
+ this.rerunXmlTag = rerunXmlTag;
+ }
+
+ public String getXmlTag()
+ {
+ return xmlTag;
+ }
+
+ public String getFlakyXmlTag()
+ {
+ return flakyXmlTag;
+ }
+
+ public String getRerunXmlTag()
+ {
+ return rerunXmlTag;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
index a3e4455..d120c11 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
@@ -19,6 +19,12 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
+import org.apache.maven.shared.utils.io.IOUtil;
+import org.apache.maven.shared.utils.xml.XMLWriter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.ReporterException;
+import org.apache.maven.surefire.report.SafeThrowable;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
@@ -27,16 +33,17 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
-import org.apache.maven.shared.utils.io.IOUtil;
-import org.apache.maven.shared.utils.xml.XMLWriter;
-import org.apache.maven.surefire.report.ReportEntry;
-import org.apache.maven.surefire.report.ReporterException;
-import org.apache.maven.surefire.report.SafeThrowable;
-
+import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars;
/**
@@ -84,46 +91,152 @@ public class StatelessXmlReporter
private final boolean trimStackTrace;
+ private final int rerunFailingTestsCount;
+
+ // Map between test class name and a map between test method name
+ // and the list of runs for each test method
+ private Map<String, Map<String, List<WrappedReportEntry>>> testClassMethodRunHistoryMap =
+ Collections.synchronizedMap( new HashMap<String, Map<String, List<WrappedReportEntry>>>() );
- public StatelessXmlReporter( File reportsDirectory, String reportNameSuffix, boolean trimStackTrace )
+ public StatelessXmlReporter( File reportsDirectory, String reportNameSuffix, boolean trimStackTrace,
+ int rerunFailingTestsCount )
{
this.reportsDirectory = reportsDirectory;
this.reportNameSuffix = reportNameSuffix;
this.trimStackTrace = trimStackTrace;
+ this.rerunFailingTestsCount = rerunFailingTestsCount;
}
public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStats testSetStats )
throws ReporterException
{
+ String testClassName = testSetReportEntry.getName();
+
+ Map<String, List<WrappedReportEntry>> methodRunHistoryMap = getAddMethodRunHistoryMap( testClassName );
+
+ // Update testClassMethodRunHistoryMap
+ for ( WrappedReportEntry methodEntry : testSetStats.getReportEntries() )
+ {
+ getAddMethodEntryList( methodRunHistoryMap, methodEntry );
+ }
FileOutputStream outputStream = getOutputStream( testSetReportEntry );
OutputStreamWriter fw = getWriter( outputStream );
try
{
-
org.apache.maven.shared.utils.xml.XMLWriter ppw =
new org.apache.maven.shared.utils.xml.PrettyPrintXMLWriter( fw );
ppw.setEncoding( ENCODING );
- createTestSuiteElement( ppw, testSetReportEntry, testSetStats, reportNameSuffix );
+ createTestSuiteElement( ppw, testSetReportEntry, testSetStats, reportNameSuffix,
+ testSetStats.elapsedTimeAsString( getRunTimeForAllTests( methodRunHistoryMap ) ) );
showProperties( ppw );
- for ( WrappedReportEntry entry : testSetStats.getReportEntries() )
+ // Iterate through all the test methods in the test class
+ for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() )
{
- if ( ReportEntryType.success.equals( entry.getReportEntryType() ) )
+ List<WrappedReportEntry> methodEntryList = entry.getValue();
+ if ( methodEntryList == null )
{
- startTestElement( ppw, entry, reportNameSuffix );
- ppw.endElement();
+ throw new IllegalStateException( "Get null test method run history" );
}
- else
+ if ( methodEntryList.size() == 0 )
{
- getTestProblems( fw, ppw, entry, trimStackTrace, reportNameSuffix, outputStream );
+ continue;
}
+ if ( rerunFailingTestsCount > 0 )
+ {
+ TestResultType resultType = getTestResultType( methodEntryList );
+ switch ( resultType )
+ {
+ case success:
+ for ( WrappedReportEntry methodEntry : methodEntryList )
+ {
+ if ( methodEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ startTestElement( ppw, methodEntry, reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ ppw.endElement();
+ }
+ }
+ break;
+ case error:
+ case failure:
+ // When rerunFailingTestsCount is set to larger than 0
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ boolean firstRun = true;
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( firstRun )
+ {
+ firstRun = false;
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getXmlTag(), false );
+ createOutErrElements( fw, ppw, singleRunEntry, outputStream );
+ }
+ else
+ {
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getRerunXmlTag(), true );
+ }
+ }
+ ppw.endElement();
+ break;
+ case flake:
+ String runtime = "";
+ // Get the run time of the first successful run
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ runtime = singleRunEntry.elapsedTimeAsString();
+ break;
+ }
+ }
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix, runtime );
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() != ReportEntryType.success )
+ {
+ getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
+ singleRunEntry.getReportEntryType().getFlakyXmlTag(), true );
+ }
+ }
+ ppw.endElement();
+
+ break;
+ case skipped:
+ startTestElement( ppw, methodEntryList.get( 0 ), reportNameSuffix,
+ methodEntryList.get( 0 ).elapsedTimeAsString() );
+ getTestProblems( fw, ppw, methodEntryList.get( 0 ), trimStackTrace, outputStream,
+ methodEntryList.get( 0 ).getReportEntryType().getXmlTag(), false );
+ ppw.endElement();
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+ else
+ {
+ // rerunFailingTestsCount is smaller than 1, but for some reasons a test could be run
+ // for more than once
+ for ( WrappedReportEntry methodEntry : methodEntryList )
+ {
+ startTestElement( ppw, methodEntry, reportNameSuffix, methodEntry.elapsedTimeAsString() );
+ if ( methodEntry.getReportEntryType() != ReportEntryType.success )
+ {
+ getTestProblems( fw, ppw, methodEntry, trimStackTrace, outputStream,
+ methodEntry.getReportEntryType().getXmlTag(), false );
+ createOutErrElements( fw, ppw, methodEntry, outputStream );
+ }
+ ppw.endElement();
+ }
+ }
}
ppw.endElement(); // TestSuite
-
}
finally
{
@@ -131,11 +244,115 @@ public class StatelessXmlReporter
}
}
+ /**
+ * Clean testClassMethodRunHistoryMap
+ */
+ public void cleanTestHistoryMap() {
+ testClassMethodRunHistoryMap.clear();
+ }
+
+ /**
+ * Get the result of a test from a list of its runs in WrappedReportEntry
+ *
+ * @param methodEntryList the list of runs for a given test
+ * @return the TestResultType for the given test
+ */
+ private static TestResultType getTestResultType( List<WrappedReportEntry> methodEntryList )
+ {
+ List<ReportEntryType> testResultTypeList = new ArrayList<ReportEntryType>();
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ testResultTypeList.add( singleRunEntry.getReportEntryType() );
+ }
+
+ return DefaultReporterFactory.getTestResultType( testResultTypeList );
+ }
+
+ /**
+ * Get run time for the entire test suite (test class)
+ * For a successful/failed/error test, the run time is the first run
+ * For a flaky test, the run time is the first successful run's time
+ * The run time for the entire test class is the sum of all its test methods
+ *
+ *
+ * @param methodRunHistoryMap the input map between test method name and the list of all its runs
+ * in a given test class
+ * @return the run time for the entire test class
+ */
+ private static int getRunTimeForAllTests( Map<String, List<WrappedReportEntry>> methodRunHistoryMap )
+ {
+ int totalTimeForSuite = 0;
+ for ( Map.Entry<String, List<WrappedReportEntry>> entry : methodRunHistoryMap.entrySet() )
+ {
+ List<WrappedReportEntry> methodEntryList = entry.getValue();
+ if ( methodEntryList == null )
+ {
+ throw new IllegalStateException( "Get null test method run history" );
+ }
+ if ( methodEntryList.size() == 0 )
+ {
+ continue;
+ }
+
+ TestResultType resultType = getTestResultType( methodEntryList );
+
+ switch ( resultType )
+ {
+ case success:
+ case error:
+ case failure:
+ // Get the first run's time for failure/error/success runs
+ totalTimeForSuite = totalTimeForSuite + methodEntryList.get( 0 ).getElapsed();
+ break;
+ case flake:
+ // Get the first successful run's time for flaky runs
+ for ( WrappedReportEntry singleRunEntry : methodEntryList )
+ {
+ if ( singleRunEntry.getReportEntryType() == ReportEntryType.success )
+ {
+ totalTimeForSuite = totalTimeForSuite + singleRunEntry.getElapsed();
+ break;
+ }
+ }
+ break;
+ case skipped:
+ break;
+ default:
+ throw new IllegalStateException( "Get unknown test result type" );
+ }
+ }
+ return totalTimeForSuite;
+ }
+
private OutputStreamWriter getWriter( FileOutputStream fos )
{
return new OutputStreamWriter( fos, ENCODING_CS );
}
+
+ private void getAddMethodEntryList( Map<String, List<WrappedReportEntry>> methodRunHistoryMap,
+ WrappedReportEntry methodEntry )
+ {
+ List<WrappedReportEntry> methodEntryList = methodRunHistoryMap.get( methodEntry.getName() );
+ if ( methodEntryList == null )
+ {
+ methodEntryList = new ArrayList<WrappedReportEntry>();
+ methodRunHistoryMap.put( methodEntry.getName(), methodEntryList );
+ }
+ methodEntryList.add( methodEntry );
+ }
+
+ private Map<String, List<WrappedReportEntry>> getAddMethodRunHistoryMap( String testClassName )
+ {
+ Map<String, List<WrappedReportEntry>> methodRunHistoryMap = testClassMethodRunHistoryMap.get( testClassName );
+ if ( methodRunHistoryMap == null )
+ {
+ methodRunHistoryMap = Collections.synchronizedMap( new LinkedHashMap<String, List<WrappedReportEntry>>() );
+ testClassMethodRunHistoryMap.put( testClassName, methodRunHistoryMap );
+ }
+ return methodRunHistoryMap;
+ }
+
private FileOutputStream getOutputStream( WrappedReportEntry testSetReportEntry )
{
File reportFile = getReportFile( testSetReportEntry, reportsDirectory, reportNameSuffix );
@@ -173,7 +390,8 @@ public class StatelessXmlReporter
return reportFile;
}
- private static void startTestElement( XMLWriter ppw, WrappedReportEntry report, String reportNameSuffix )
+ private static void startTestElement( XMLWriter ppw, WrappedReportEntry report, String reportNameSuffix,
+ String timeAsString )
{
ppw.startElement( "testcase" );
ppw.addAttribute( "name", report.getReportName() );
@@ -192,11 +410,11 @@ public class StatelessXmlReporter
ppw.addAttribute( "classname", report.getSourceName() );
}
}
- ppw.addAttribute( "time", report.elapsedTimeAsString() );
+ ppw.addAttribute( "time", timeAsString );
}
private static void createTestSuiteElement( XMLWriter ppw, WrappedReportEntry report, TestSetStats testSetStats,
- String reportNameSuffix1 )
+ String reportNameSuffix1, String timeAsString )
{
ppw.startElement( "testsuite" );
@@ -207,7 +425,7 @@ public class StatelessXmlReporter
ppw.addAttribute( "group", report.getGroup() );
}
- ppw.addAttribute( "time", testSetStats.getElapsedForTestSet() );
+ ppw.addAttribute( "time", timeAsString );
ppw.addAttribute( "tests", String.valueOf( testSetStats.getCompletedCount() ) );
@@ -221,12 +439,10 @@ public class StatelessXmlReporter
private void getTestProblems( OutputStreamWriter outputStreamWriter, XMLWriter ppw, WrappedReportEntry report,
- boolean trimStackTrace, String reportNameSuffix, FileOutputStream fw )
+ boolean trimStackTrace, FileOutputStream fw,
+ String testErrorType, boolean createOutErrElementsInside )
{
-
- startTestElement( ppw, report, reportNameSuffix );
-
- ppw.startElement( report.getReportEntryType().name() );
+ ppw.startElement( testErrorType );
String stackTrace = report.getStackTrace( trimStackTrace );
@@ -259,15 +475,21 @@ public class StatelessXmlReporter
ppw.writeText( extraEscape( stackTrace, false ) );
}
+ if ( createOutErrElementsInside )
+ {
+ createOutErrElements( outputStreamWriter, ppw, report, fw );
+ }
+
ppw.endElement(); // entry type
+ }
+ // Create system-out and system-err elements
+ private void createOutErrElements( OutputStreamWriter outputStreamWriter, XMLWriter ppw, WrappedReportEntry report,
+ FileOutputStream fw )
+ {
EncodingOutputStream eos = new EncodingOutputStream( fw );
-
addOutputStreamElement( outputStreamWriter, fw, eos, ppw, report.getStdout(), "system-out" );
-
addOutputStreamElement( outputStreamWriter, fw, eos, ppw, report.getStdErr(), "system-err" );
-
- ppw.endElement(); // test element
}
private void addOutputStreamElement( OutputStreamWriter outputStreamWriter, OutputStream fw,
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
new file mode 100644
index 0000000..e202a48
--- /dev/null
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
@@ -0,0 +1,60 @@
+package org.apache.maven.plugin.surefire.report;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+/**
+ *
+ * Maintains per-thread test result state for a single test method.
+ *
+ * @author Qingzhou Luo
+ *
+ */
+public class TestMethodStats
+{
+ private final String testClassMethodName;
+
+ private final ReportEntryType resultType;
+
+ private final StackTraceWriter stackTraceWriter;
+
+ public TestMethodStats( String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter )
+ {
+ this.testClassMethodName = testClassMethodName;
+ this.resultType = resultType;
+ this.stackTraceWriter = stackTraceWriter;
+ }
+
+ public String getTestClassMethodName()
+ {
+ return testClassMethodName;
+ }
+
+ public ReportEntryType getResultType()
+ {
+ return resultType;
+ }
+
+ public StackTraceWriter getStackTraceWriter()
+ {
+ return stackTraceWriter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
index c100aae..ca2403d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
@@ -19,16 +19,14 @@ package org.apache.maven.plugin.surefire.report;
* under the License.
*/
-import org.apache.commons.io.output.DeferredFileOutputStream;
import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
import org.apache.maven.surefire.report.ConsoleLogger;
import org.apache.maven.surefire.report.ConsoleOutputReceiver;
import org.apache.maven.surefire.report.ReportEntry;
import org.apache.maven.surefire.report.RunListener;
-import org.apache.maven.surefire.report.RunStatistics;
-import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -40,10 +38,10 @@ import java.util.List;
public class TestSetRunListener
implements RunListener, ConsoleOutputReceiver, ConsoleLogger
{
- private final RunStatistics globalStatistics;
-
private final TestSetStats detailsForThis;
+ private List<TestMethodStats> testMethodStats;
+
private Utf8RecodingDeferredFileOutputStream testStdOut = initDeferred( "stdout" );
private Utf8RecodingDeferredFileOutputStream testStdErr = initDeferred( "stderr" );
@@ -68,7 +66,7 @@ public class TestSetRunListener
public TestSetRunListener( ConsoleReporter consoleReporter, FileReporter fileReporter,
StatelessXmlReporter simpleXMLReporter,
TestcycleConsoleOutputReceiver consoleOutputReceiver,
- StatisticsReporter statisticsReporter, RunStatistics globalStats, boolean trimStackTrace,
+ StatisticsReporter statisticsReporter, boolean trimStackTrace,
boolean isPlainFormat, boolean briefOrPlainFormat )
{
this.consoleReporter = consoleReporter;
@@ -78,7 +76,7 @@ public class TestSetRunListener
this.consoleOutputReceiver = consoleOutputReceiver;
this.briefOrPlainFormat = briefOrPlainFormat;
this.detailsForThis = new TestSetStats( trimStackTrace, isPlainFormat );
- this.globalStatistics = globalStats;
+ this.testMethodStats = new ArrayList<TestMethodStats>( );
}
public void info( String message )
@@ -154,7 +152,7 @@ public class TestSetRunListener
wrap.getStdout().free();
wrap.getStdErr().free();
- globalStatistics.add( detailsForThis );
+ addTestMethodStats();
detailsForThis.reset();
}
@@ -188,7 +186,6 @@ public class TestSetRunListener
{
statisticsReporter.testError( reportEntry );
}
- globalStatistics.addErrorSource( reportEntry.getStackTraceWriter() );
clearCapture();
}
@@ -200,7 +197,6 @@ public class TestSetRunListener
{
statisticsReporter.testFailed( reportEntry );
}
- globalStatistics.addFailureSource( reportEntry.getStackTraceWriter() );
clearCapture();
}
@@ -260,4 +256,20 @@ public class TestSetRunListener
consoleOutputReceiver.close();
}
}
+
+ public void addTestMethodStats()
+ {
+ for (WrappedReportEntry reportEntry : detailsForThis.getReportEntries())
+ {
+ TestMethodStats methodStats =
+ new TestMethodStats( reportEntry.getClassMethodName(), reportEntry.getReportEntryType(),
+ reportEntry.getStackTraceWriter() );
+ testMethodStats.add( methodStats );
+ }
+ }
+
+ public List<TestMethodStats> getTestMethodStats()
+ {
+ return testMethodStats;
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
index b8078b6..a175b0a 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java
@@ -154,17 +154,17 @@ public class TestSetStats
return skipped;
}
+ public String elapsedTimeAsString( long runTime )
+ {
+ return numberFormat.format( (double) runTime / MS_PER_SEC );
+ }
+
private static final String TEST_SET_COMPLETED_PREFIX = "Tests run: ";
private final NumberFormat numberFormat = NumberFormat.getInstance( Locale.ENGLISH );
private static final int MS_PER_SEC = 1000;
- String elapsedTimeAsString( long runTime )
- {
- return numberFormat.format( (double) runTime / MS_PER_SEC );
- }
-
public String getElapsedForTestSet()
{
return elapsedTimeAsString( elapsedForTestSet );
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
index d38ae76..ef6961d 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/WrappedReportEntry.java
@@ -87,6 +87,11 @@ public class WrappedReportEntry
return original.getName();
}
+ public String getClassMethodName()
+ {
+ return getSourceName() + "." + getName();
+ }
+
public String getGroup()
{
return original.getGroup();
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
index 139c3d7..926658f 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
@@ -19,27 +19,21 @@ package org.apache.maven.surefire.report;
* under the License.
*/
+import org.apache.maven.plugin.surefire.report.TestSetStats;
+import org.apache.maven.surefire.suite.RunResult;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import org.apache.maven.plugin.surefire.report.TestSetStats;
-import org.apache.maven.surefire.suite.RunResult;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
/**
* @author Kristian Rosenvold
*/
public class RunStatistics
{
- /**
- * Holds the source(s) that causes the error(s).
- */
- private final Sources errorSources = new Sources();
-
- /**
- * Holds the source(s) that causes the failure(s).
- */
- private final Sources failureSources = new Sources();
-
private int completedCount;
private int errors;
@@ -48,53 +42,45 @@ public class RunStatistics
private int skipped;
+ private int flakes;
- public void addErrorSource( StackTraceWriter stackTraceWriter )
+ public RunStatistics()
{
- if ( stackTraceWriter == null )
- {
- throw new IllegalArgumentException( "Cant be null" );
- }
- errorSources.addSource( stackTraceWriter );
}
- public void addFailureSource( StackTraceWriter stackTraceWriter )
+ public synchronized boolean hadFailures()
{
- if ( stackTraceWriter == null )
- {
- throw new IllegalArgumentException( "Cant be null" );
- }
- failureSources.addSource( stackTraceWriter );
+ return failures > 0;
}
- public Collection<String> getErrorSources()
+ public synchronized boolean hadErrors()
{
- return errorSources.getListOfSources();
+ return errors > 0;
}
- public Collection<String> getFailureSources()
+ public synchronized int getCompletedCount()
{
- return failureSources.getListOfSources();
+ return completedCount;
}
- public synchronized boolean hadFailures()
+ public synchronized int getSkipped()
{
- return failures > 0;
+ return skipped;
}
- public synchronized boolean hadErrors()
+ public synchronized int getFailures()
{
- return errors > 0;
+ return failures;
}
- public synchronized int getCompletedCount()
+ public synchronized int getErrors()
{
- return completedCount;
+ return errors;
}
- public synchronized int getSkipped()
+ public synchronized int getFlakes()
{
- return skipped;
+ return flakes;
}
public synchronized void add( TestSetStats testSetStats )
@@ -105,41 +91,29 @@ public class RunStatistics
this.skipped += testSetStats.getSkipped();
}
- public synchronized RunResult getRunResult()
+ public synchronized void set( int completedCount, int errors, int failures, int skipped, int flakes )
{
- return new RunResult( completedCount, errors, failures, skipped );
+ this.completedCount = completedCount;
+ this.errors = errors;
+ this.failures = failures;
+ this.skipped = skipped;
+ this.flakes = flakes;
}
- public synchronized String getSummary()
+ public synchronized RunResult getRunResult()
{
- return "Tests run: " + completedCount + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: "
- + skipped;
+ return new RunResult( completedCount, errors, failures, skipped, flakes );
}
-
- private static class Sources
+ public synchronized String getSummary()
{
- private final Collection<String> listOfSources = new ArrayList<String>();
-
- void addSource( String source )
- {
- synchronized ( listOfSources )
- {
- listOfSources.add( source );
- }
- }
-
- void addSource( StackTraceWriter stackTraceWriter )
- {
- addSource( stackTraceWriter.smartTrimmedStackTrace() );
- }
-
- Collection<String> getListOfSources()
+ String summary =
+ "Tests run: " + completedCount + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: "
+ + skipped;
+ if ( flakes > 0 )
{
- synchronized ( listOfSources )
- {
- return Collections.unmodifiableCollection( listOfSources );
- }
+ summary += ", Flakes: " + flakes;
}
+ return summary;
}
}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
index c11ce59..f72d4cd 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
@@ -57,6 +57,8 @@ public class BooterDeserializerProviderConfigurationTest
private final String aUserRequestedTest = "aUserRequestedTest";
+ private final int rerunFailingTestsCount = 3;
+
private static ClassLoaderConfiguration getForkConfiguration()
{
return new ClassLoaderConfiguration( true, false );
@@ -124,6 +126,7 @@ public class BooterDeserializerProviderConfigurationTest
Assert.assertEquals( expected[1], suiteXmlFiles.get( 1 ) );
Assert.assertEquals( getTestSourceDirectory(), testSuiteDefinition.getTestSourceDirectory() );
Assert.assertEquals( aUserRequestedTest, testSuiteDefinition.getRequestedTest() );
+ Assert.assertEquals( rerunFailingTestsCount, testSuiteDefinition.getRerunFailingTestsCount() );
}
public void testTestForFork()
@@ -220,7 +223,7 @@ public class BooterDeserializerProviderConfigurationTest
String aUserRequestedTestMethod = "aUserRequestedTestMethod";
TestRequest testSuiteDefinition =
new TestRequest( getSuiteXmlFileStrings(), getTestSourceDirectory(), aUserRequestedTest,
- aUserRequestedTestMethod );
+ aUserRequestedTestMethod, rerunFailingTestsCount );
RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null );
return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration,
new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new Properties(),
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/fefaae7f/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
----------------------------------------------------------------------
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
new file mode 100644
index 0000000..0dc7865
--- /dev/null
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -0,0 +1,206 @@
+package org.apache.maven.plugin.surefire.report;
+
+import junit.framework.TestCase;
+import org.apache.maven.plugin.surefire.StartupReportConfiguration;
+import org.apache.maven.surefire.report.DefaultDirectConsoleReporter;
+import org.apache.maven.surefire.report.ReportEntry;
+import org.apache.maven.surefire.report.RunStatistics;
+import org.apache.maven.surefire.report.SafeThrowable;
+import org.apache.maven.surefire.report.StackTraceWriter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DefaultReporterFactoryTest
+ extends TestCase
+{
+
+ private final static String TEST_ONE = "testOne";
+
+ private final static String TEST_TWO = "testTwo";
+
+ private final static String TEST_THREE = "testThree";
+
+ private final static String TEST_FOUR = "testFour";
+
+ private final static String TEST_FIVE = "testFive";
+
+ private final static String ASSERTION_FAIL = "assertionFail";
+
+ private final static String ERROR = "error";
+
+ public void testMergeTestHistoryResult()
+ {
+ DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() );
+
+ // First run, four tests failed and one passed
+ List<TestMethodStats> firstRunStats = new ArrayList<TestMethodStats>();
+ firstRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ firstRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_FOUR, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ firstRunStats.add(
+ new TestMethodStats( TEST_FIVE, ReportEntryType.success, null ) );
+
+ // Second run, two tests passed
+ List<TestMethodStats> secondRunStats = new ArrayList<TestMethodStats>();
+ secondRunStats.add(
+ new TestMethodStats( TEST_ONE, ReportEntryType.failure, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
+ secondRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.success, null ) );
+ secondRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+ secondRunStats.add( new TestMethodStats( TEST_FOUR, ReportEntryType.success, null ) );
+
+ // Third run, another test passed
+ List<TestMethodStats> thirdRunStats = new ArrayList<TestMethodStats>();
+ thirdRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.success, null ) );
+ thirdRunStats.add(
+ new TestMethodStats( TEST_THREE, ReportEntryType.error, new DummyStackTraceWriter( ERROR ) ) );
+
+ TestSetRunListener firstRunListener = mock( TestSetRunListener.class );
+ TestSetRunListener secondRunListener = mock( TestSetRunListener.class );
+ TestSetRunListener thirdRunListener = mock( TestSetRunListener.class );
+ when( firstRunListener.getTestMethodStats() ).thenReturn( firstRunStats );
+ when( secondRunListener.getTestMethodStats() ).thenReturn( secondRunStats );
+ when( thirdRunListener.getTestMethodStats() ).thenReturn( thirdRunStats );
+
+ factory.addListener( firstRunListener );
+ factory.addListener( secondRunListener );
+ factory.addListener( thirdRunListener );
+
+ factory.mergeTestHistoryResult();
+ RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
+
+ // Only TEST_THREE is a failing test, other three are flaky tests
+ assertEquals( 5, mergedStatistics.getCompletedCount() );
+ assertEquals( 0, mergedStatistics.getErrors() );
+ assertEquals( 1, mergedStatistics.getFailures() );
+ assertEquals( 3, mergedStatistics.getFlakes() );
+ assertEquals( 0, mergedStatistics.getSkipped() );
+
+ // Now test the result will be printed out correctly
+ DummyTestReporter reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.flake );
+ String[] expectedFlakeOutput =
+ { "Flaked tests: ", TEST_FOUR, " Run 1: " + ASSERTION_FAIL, " Run 2: PASS", "", TEST_ONE,
+ " Run 1: " + ERROR, " Run 2: " + ASSERTION_FAIL, " Run 3: PASS", "", TEST_TWO, " Run 1: " + ERROR,
+ " Run 2: PASS", "", "" };
+ assertEquals( Arrays.asList( expectedFlakeOutput ), reporter.getMessages() );
+
+ reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.failure );
+ String[] expectedFailureOutput =
+ { "Failed tests: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, "",
+ "" };
+ assertEquals( Arrays.asList( expectedFailureOutput ), reporter.getMessages() );
+
+ reporter = new DummyTestReporter();
+ factory.printTestFailures( reporter, DefaultReporterFactory.TestResultType.error );
+ String[] expectedErrorOutput = { "" };
+ assertEquals( Arrays.asList( expectedErrorOutput ), reporter.getMessages() );
+ }
+
+ static class DummyTestReporter
+ extends DefaultDirectConsoleReporter
+ {
+
+ private final List<String> messages = new ArrayList<String>();
+
+ public DummyTestReporter()
+ {
+ super( System.out );
+ }
+
+ @Override
+ public void info( String msg )
+ {
+ messages.add( msg );
+ }
+
+ public List<String> getMessages()
+ {
+ return messages;
+ }
+ }
+
+ public void testGetTestResultType()
+ {
+ DefaultReporterFactory factory = new DefaultReporterFactory( StartupReportConfiguration.defaultValue() );
+
+ List<ReportEntryType> emptyList = new ArrayList<ReportEntryType>();
+ assertEquals( unknown, factory.getTestResultType( emptyList ) );
+
+ List<ReportEntryType> successList = new ArrayList<ReportEntryType>();
+ successList.add( ReportEntryType.success );
+ successList.add( ReportEntryType.success );
+ assertEquals( success, factory.getTestResultType( successList ) );
+
+ List<ReportEntryType> failureErrorList = new ArrayList<ReportEntryType>();
+ failureErrorList.add( ReportEntryType.failure );
+ failureErrorList.add( ReportEntryType.error );
+ assertEquals( failure, factory.getTestResultType( failureErrorList ) );
+
+ List<ReportEntryType> errorFailureList = new ArrayList<ReportEntryType>();
+ errorFailureList.add( ReportEntryType.error );
+ errorFailureList.add( ReportEntryType.failure );
+ assertEquals( error, factory.getTestResultType( errorFailureList ) );
+
+ List<ReportEntryType> flakeList = new ArrayList<ReportEntryType>();
+ flakeList.add( ReportEntryType.success );
+ flakeList.add( ReportEntryType.failure );
+ assertEquals( flake, factory.getTestResultType( flakeList ) );
+
+ flakeList = new ArrayList<ReportEntryType>();
+ flakeList.add( ReportEntryType.error );
+ flakeList.add( ReportEntryType.success );
+ flakeList.add( ReportEntryType.failure );
+ assertEquals( flake, factory.getTestResultType( flakeList ) );
+
+ List<ReportEntryType> skippedList = new ArrayList<ReportEntryType>();
+ skippedList.add( ReportEntryType.skipped );
+ assertEquals( skipped, factory.getTestResultType( skippedList ) );
+ }
+
+ static class DummyStackTraceWriter
+ implements StackTraceWriter
+ {
+
+ private final String stackTrace;
+
+ public DummyStackTraceWriter( String stackTrace )
+ {
+ this.stackTrace = stackTrace;
+ }
+
+ @Override
+ public String writeTraceToString()
+ {
+ return "";
+ }
+
+ @Override
+ public String writeTrimmedTraceToString()
+ {
+ return "";
+ }
+
+ @Override
+ public String smartTrimmedStackTrace()
+ {
+ return stackTrace;
+ }
+
+ @Override
+ public SafeThrowable getThrowable()
+ {
+ return null;
+ }
+ }
+}
\ No newline at end of file