You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2023/11/05 19:45:15 UTC
(maven-surefire) 01/01: [SUREFIRE-1656] Print Total Elapsed Time of Tests
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch SUREFIRE-1656
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
commit fd4c3d2628f8575a9872f5d254b99990ee550153
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Sun Nov 5 20:44:58 2023 +0100
[SUREFIRE-1656] Print Total Elapsed Time of Tests
---
.../failsafe/util/FailsafeSummaryXmlUtils.java | 7 ++++
.../maven/plugin/failsafe/RunResultTest.java | 8 +++--
.../surefire/report/DefaultReporterFactory.java | 10 ++++--
.../plugin/surefire/report/TestMethodStats.java | 9 ++++-
.../plugin/surefire/report/TestSetRunListener.java | 3 +-
.../maven/surefire/report/RunStatistics.java | 23 ++++++++++++-
.../report/DefaultReporterFactoryTest.java | 29 ++++++++--------
.../maven/surefire/report/RunStatisticsTest.java | 3 +-
.../apache/maven/surefire/api/suite/RunResult.java | 39 ++++++++++++++++++----
9 files changed, 102 insertions(+), 29 deletions(-)
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
index ccd826c20..80655d52b 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
@@ -35,6 +35,7 @@ import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import static java.lang.Boolean.parseBoolean;
+import static java.lang.Float.parseFloat;
import static java.lang.Integer.parseInt;
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -56,6 +57,7 @@ public final class FailsafeSummaryXmlUtils {
private static final String MESSAGE_ELEMENT = "<failureMessage>%s</failureMessage>";
+ // TODO What happened to flakes?
private static final String FAILSAFE_SUMMARY_XML_TEMPLATE = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<failsafe-summary xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\""
+ " xsi:noNamespaceSchemaLocation=\"" + FAILSAFE_SUMMARY_XML_SCHEMA_LOCATION + "\""
@@ -64,6 +66,7 @@ public final class FailsafeSummaryXmlUtils {
+ " <errors>%d</errors>\n"
+ " <failures>%d</failures>\n"
+ " <skipped>%d</skipped>\n"
+ + " <time>%s</time>\n"
+ " %s\n"
+ "</failsafe-summary>";
@@ -82,6 +85,7 @@ public final class FailsafeSummaryXmlUtils {
String errors = xpath.evaluate("/failsafe-summary/errors", root);
String failures = xpath.evaluate("/failsafe-summary/failures", root);
String skipped = xpath.evaluate("/failsafe-summary/skipped", root);
+ String elapsed = xpath.evaluate("/failsafe-summary/time", root);
String failureMessage = xpath.evaluate("/failsafe-summary/failureMessage", root);
String timeout = xpath.evaluate("/failsafe-summary/@timeout", root);
@@ -90,6 +94,8 @@ public final class FailsafeSummaryXmlUtils {
parseInt(errors),
parseInt(failures),
parseInt(skipped),
+ 0,
+ isBlank(elapsed) ? null : ((int) parseFloat(elapsed) * 1000),
isBlank(failureMessage) ? null : unescapeXml(failureMessage),
parseBoolean(timeout));
}
@@ -107,6 +113,7 @@ public final class FailsafeSummaryXmlUtils {
fromRunResult.getErrors(),
fromRunResult.getFailures(),
fromRunResult.getSkipped(),
+ fromRunResult.getElapsed() != null ? String.valueOf(fromRunResult.getElapsed() / 1000.0f) : "",
msg);
try (FileOutputStream os = new FileOutputStream(toFailsafeSummaryXml)) {
diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
index 1bd4145bc..8c42ddd4b 100644
--- a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
+++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
@@ -56,18 +56,18 @@ public class RunResultTest {
@Test
public void testFailures() throws Exception {
- writeReadCheck(new RunResult(0, 1, 2, 3, "stacktraceHere", false));
+ writeReadCheck(new RunResult(0, 1, 2, 3, 220, "stacktraceHere", false));
}
@Test
public void testSkipped() throws Exception {
- writeReadCheck(new RunResult(3, 2, 1, 0, null, true));
+ writeReadCheck(new RunResult(3, 2, 1, 0, 50, null, true));
}
@Test
public void testAppendSerialization() throws Exception {
RunResult simpleAggregate = getSimpleAggregate();
- RunResult additional = new RunResult(2, 1, 2, 2, "msg " + ((char) 0x0E01), true);
+ RunResult additional = new RunResult(2, 1, 2, 2, 500, "msg " + ((char) 0x0E01), true);
File summary = SureFireFileManager.createTempFile("failsafe", "test");
FailsafeSummaryXmlUtils.writeSummary(simpleAggregate, summary, false);
@@ -88,6 +88,8 @@ public class RunResultTest {
assertThat(expected.getFlakes()).isEqualTo(2);
+ assertThat(expected.getElapsed()).isEqualTo(500);
+
assertThat(expected.getFailure()).isEqualTo("msg " + ((char) 0x0E01));
assertThat(expected.isTimeout()).isTrue();
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
index 762e1348c..66f205a94 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -263,7 +263,7 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
}
// Update globalStatistics by iterating through mergedTestHistoryResult
- int completedCount = 0, skipped = 0;
+ int completedCount = 0, skipped = 0, elapsed = -1;
for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
List<TestMethodStats> testMethodStats = entry.getValue();
@@ -273,6 +273,12 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
List<ReportEntryType> resultTypes = new ArrayList<>();
for (TestMethodStats methodStats : testMethodStats) {
resultTypes.add(methodStats.getResultType());
+ if (methodStats.getElapsed() != null) {
+ if (elapsed == -1) {
+ elapsed = 0;
+ }
+ elapsed += methodStats.getElapsed();
+ }
}
switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
@@ -303,7 +309,7 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
}
}
- globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
+ globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size(), (elapsed != -1 ? elapsed : null));
}
/**
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
index d3abb53a7..6033509c9 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
@@ -34,10 +34,13 @@ public class TestMethodStats {
private final StackTraceWriter stackTraceWriter;
- public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter) {
+ private final Integer elapsed;
+
+ public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter, Integer elapsed) {
this.testClassMethodName = testClassMethodName;
this.resultType = resultType;
this.stackTraceWriter = stackTraceWriter;
+ this.elapsed = elapsed;
}
public String getTestClassMethodName() {
@@ -51,4 +54,8 @@ public class TestMethodStats {
public StackTraceWriter getStackTraceWriter() {
return stackTraceWriter;
}
+
+ public Integer getElapsed() {
+ return elapsed;
+ }
}
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
index 5e90fb055..391bc0d94 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
@@ -288,7 +288,8 @@ public class TestSetRunListener implements TestReportListener<TestOutputReportEn
TestMethodStats methodStats = new TestMethodStats(
reportEntry.getClassMethodName(),
reportEntry.getReportEntryType(),
- reportEntry.getStackTraceWriter());
+ reportEntry.getStackTraceWriter(),
+ reportEntry.getElapsed());
testMethodStats.add(methodStats);
}
}
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
index 00abbf248..6436b8892 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
@@ -18,12 +18,25 @@
*/
package org.apache.maven.surefire.report;
+import java.text.MessageFormat;
+import java.util.Locale;
+
import org.apache.maven.surefire.api.suite.RunResult;
/**
* @author Kristian Rosenvold
*/
public final class RunStatistics {
+ private static final float ONE_SECOND = 1000.0f;
+
+ /*
+ * Rationale: The idea is to always display four digits for visually consistent output
+ * Important: Keep in sync with maven-surefire-report-plugin/src/main/resources/surefire-report.properties
+ */
+ private final MessageFormat elapsedTimeFormat = new MessageFormat(
+ "{0,choice,0#0|0.0<{0,number,0.000}|10#{0,number,0.00}|100#{0,number,0.0}|1000#{0,number,0}} s",
+ Locale.ROOT);
+
private int completedCount;
private int errors;
@@ -34,6 +47,8 @@ public final class RunStatistics {
private int flakes;
+ private Integer elapsed;
+
public synchronized int getCompletedCount() {
return completedCount;
}
@@ -54,12 +69,17 @@ public final class RunStatistics {
return flakes;
}
- public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes) {
+ public synchronized Integer getElapsed() {
+ return elapsed;
+ }
+
+ public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) {
this.completedCount = completedCount;
this.errors = errors;
this.failures = failures;
this.skipped = skipped;
this.flakes = flakes;
+ this.elapsed = elapsed;
}
public synchronized RunResult getRunResult() {
@@ -72,6 +92,7 @@ public final class RunStatistics {
if (flakes > 0) {
summary += ", Flakes: " + flakes;
}
+ summary += ", Time elapsed: " + (elapsed != null ? elapsedTimeFormat.format(new Object[] {elapsed / ONE_SECOND}): "(unknown)");
return summary;
}
}
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
index 9d4c9f924..11e5e59c1 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -72,6 +72,7 @@ public class DefaultReporterFactoryTest extends TestCase {
private static final String ERROR = "error";
+ @SuppressWarnings("checkstyle:MagicNumber")
public void testMergeTestHistoryResult() throws Exception {
MessageUtils.setColorEnabled(false);
File target = new File(System.getProperty("user.dir"), "target");
@@ -100,26 +101,28 @@ public class DefaultReporterFactoryTest extends TestCase {
// First run, four tests failed and one passed
Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
- firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
- firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
+ //CHECKSTYLE:OFF
+ firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
+ firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
firstRunStats.add(
- new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
+ new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
firstRunStats.add(
- new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
- firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
+ new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
+ firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null, 50));
// Second run, two tests passed
Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
secondRunStats.add(
- new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
- secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
- secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
- secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
+ new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
+ secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null, 25));
+ secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), 50));
+ secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null, 25));
// Third run, another test passed
Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
- thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
- thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
+ thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null, 10));
+ thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
+ //CHECKSTYLE:ON
TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
@@ -402,7 +405,7 @@ public class DefaultReporterFactoryTest extends TestCase {
assertEquals(0, statistics.getErrors());
assertEquals(0, statistics.getSkipped());
assertEquals(0, statistics.getFlakes());
- assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
+ assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", statistics.getSummary());
assertEquals(0, statistics.getCompletedCount());
List<String> messages = reporter.getMessages();
@@ -413,7 +416,7 @@ public class DefaultReporterFactoryTest extends TestCase {
assertEquals("", messages.get(4));
assertEquals("Results:", messages.get(5));
assertEquals("", messages.get(6));
- assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
+ assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", messages.get(7));
assertEquals("", messages.get(8));
assertEquals(9, messages.size());
}
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
index a3206e00a..eb6861966 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
@@ -23,10 +23,11 @@ import junit.framework.TestCase;
/**
*
*/
+@SuppressWarnings("checkstyle:MagicNumber")
public class RunStatisticsTest extends TestCase {
public void testSetRunStatistics() {
RunStatistics statistics = new RunStatistics();
- statistics.set(10, 5, 2, 1, 2);
+ statistics.set(10, 5, 2, 1, 2, 5500);
assertEquals(10, statistics.getCompletedCount());
assertEquals(5, statistics.getErrors());
assertEquals(2, statistics.getFailures());
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
index ceb182aee..fee341212 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
@@ -40,6 +40,8 @@ public class RunResult {
private final int flakes;
+ private final Integer elapsed;
+
private final String failure;
private final boolean timeout;
@@ -58,37 +60,44 @@ public class RunResult {
return errorCode(accumulatedAtTimeout, getStackTrace(cause), accumulatedAtTimeout.isTimeout());
}
+ // TODO What happened to flakes?
private static RunResult errorCode(RunResult other, String failure, boolean timeout) {
return new RunResult(
other.getCompletedCount(),
other.getErrors(),
other.getFailures(),
other.getSkipped(),
+ other.getElapsed(),
failure,
timeout);
}
public RunResult(int completedCount, int errors, int failures, int skipped) {
- this(completedCount, errors, failures, skipped, null, false);
+ this(completedCount, errors, failures, skipped, null, null, false);
}
public RunResult(int completedCount, int errors, int failures, int skipped, int flakes) {
- this(completedCount, errors, failures, skipped, flakes, null, false);
+ this(completedCount, errors, failures, skipped, flakes, null, null, false);
+ }
+
+ public RunResult(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) {
+ this(completedCount, errors, failures, skipped, flakes, elapsed, null, false);
}
- public RunResult(int completedCount, int errors, int failures, int skipped, String failure, boolean timeout) {
- this(completedCount, errors, failures, skipped, 0, failure, timeout);
+ public RunResult(int completedCount, int errors, int failures, int skipped, Integer elapsed, String failure, boolean timeout) {
+ this(completedCount, errors, failures, skipped, 0, elapsed, failure, timeout);
}
public RunResult(
- int completedCount, int errors, int failures, int skipped, int flakes, String failure, boolean timeout) {
+ int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed, String failure, boolean timeout) {
this.completedCount = completedCount;
this.errors = errors;
this.failures = failures;
this.skipped = skipped;
+ this.flakes = flakes;
+ this.elapsed = elapsed;
this.failure = failure;
this.timeout = timeout;
- this.flakes = flakes;
}
private static String getStackTrace(Exception e) {
@@ -122,6 +131,10 @@ public class RunResult {
return skipped;
}
+ public Integer getElapsed() {
+ return elapsed;
+ }
+
public Integer getFailsafeCode() // Only used for compatibility reasons.
{
if (completedCount == 0) {
@@ -167,13 +180,24 @@ public class RunResult {
int ign = getSkipped() + other.getSkipped();
int err = getErrors() + other.getErrors();
int flakes = getFlakes() + other.getFlakes();
- return new RunResult(completed, err, fail, ign, flakes, failureMessage, timeout);
+ Integer elapsed;
+ if (getElapsed() == null) {
+ elapsed = other.getElapsed();
+ } else {
+ if (other.getElapsed() == null) {
+ elapsed = getElapsed();
+ } else {
+ elapsed = getElapsed() + other.getElapsed();
+ }
+ }
+ return new RunResult(completed, err, fail, ign, flakes, elapsed, failureMessage, timeout);
}
public static RunResult noTestsRun() {
return new RunResult(0, 0, 0, 0);
}
+ // TODO What happened to flakes?
@Override
@SuppressWarnings("RedundantIfStatement")
public boolean equals(Object o) {
@@ -208,6 +232,7 @@ public class RunResult {
return true;
}
+ // TODO What happened to flakes?
@Override
public int hashCode() {
int result = completedCount;