You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2023/11/05 19:45:15 UTC

(maven-surefire) 01/01: [SUREFIRE-1656] Print Total Elapsed Time of Tests

This is an automated email from the ASF dual-hosted git repository.

michaelo pushed a commit to branch SUREFIRE-1656
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git

commit fd4c3d2628f8575a9872f5d254b99990ee550153
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Sun Nov 5 20:44:58 2023 +0100

    [SUREFIRE-1656] Print Total Elapsed Time of Tests
---
 .../failsafe/util/FailsafeSummaryXmlUtils.java     |  7 ++++
 .../maven/plugin/failsafe/RunResultTest.java       |  8 +++--
 .../surefire/report/DefaultReporterFactory.java    | 10 ++++--
 .../plugin/surefire/report/TestMethodStats.java    |  9 ++++-
 .../plugin/surefire/report/TestSetRunListener.java |  3 +-
 .../maven/surefire/report/RunStatistics.java       | 23 ++++++++++++-
 .../report/DefaultReporterFactoryTest.java         | 29 ++++++++--------
 .../maven/surefire/report/RunStatisticsTest.java   |  3 +-
 .../apache/maven/surefire/api/suite/RunResult.java | 39 ++++++++++++++++++----
 9 files changed, 102 insertions(+), 29 deletions(-)

diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
index ccd826c20..80655d52b 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java
@@ -35,6 +35,7 @@ import org.w3c.dom.Node;
 import org.xml.sax.InputSource;
 
 import static java.lang.Boolean.parseBoolean;
+import static java.lang.Float.parseFloat;
 import static java.lang.Integer.parseInt;
 import static java.lang.String.format;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -56,6 +57,7 @@ public final class FailsafeSummaryXmlUtils {
 
     private static final String MESSAGE_ELEMENT = "<failureMessage>%s</failureMessage>";
 
+    // TODO What happened to flakes?
     private static final String FAILSAFE_SUMMARY_XML_TEMPLATE = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
             + "<failsafe-summary xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\""
             + " xsi:noNamespaceSchemaLocation=\"" + FAILSAFE_SUMMARY_XML_SCHEMA_LOCATION + "\""
@@ -64,6 +66,7 @@ public final class FailsafeSummaryXmlUtils {
             + "    <errors>%d</errors>\n"
             + "    <failures>%d</failures>\n"
             + "    <skipped>%d</skipped>\n"
+            + "    <time>%s</time>\n"
             + "    %s\n"
             + "</failsafe-summary>";
 
@@ -82,6 +85,7 @@ public final class FailsafeSummaryXmlUtils {
             String errors = xpath.evaluate("/failsafe-summary/errors", root);
             String failures = xpath.evaluate("/failsafe-summary/failures", root);
             String skipped = xpath.evaluate("/failsafe-summary/skipped", root);
+            String elapsed = xpath.evaluate("/failsafe-summary/time", root);
             String failureMessage = xpath.evaluate("/failsafe-summary/failureMessage", root);
             String timeout = xpath.evaluate("/failsafe-summary/@timeout", root);
 
@@ -90,6 +94,8 @@ public final class FailsafeSummaryXmlUtils {
                     parseInt(errors),
                     parseInt(failures),
                     parseInt(skipped),
+                    0,
+                    isBlank(elapsed) ? null : ((int) parseFloat(elapsed) * 1000),
                     isBlank(failureMessage) ? null : unescapeXml(failureMessage),
                     parseBoolean(timeout));
         }
@@ -107,6 +113,7 @@ public final class FailsafeSummaryXmlUtils {
                 fromRunResult.getErrors(),
                 fromRunResult.getFailures(),
                 fromRunResult.getSkipped(),
+                fromRunResult.getElapsed() != null ? String.valueOf(fromRunResult.getElapsed() / 1000.0f) : "",
                 msg);
 
         try (FileOutputStream os = new FileOutputStream(toFailsafeSummaryXml)) {
diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
index 1bd4145bc..8c42ddd4b 100644
--- a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
+++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java
@@ -56,18 +56,18 @@ public class RunResultTest {
 
     @Test
     public void testFailures() throws Exception {
-        writeReadCheck(new RunResult(0, 1, 2, 3, "stacktraceHere", false));
+        writeReadCheck(new RunResult(0, 1, 2, 3, 220, "stacktraceHere", false));
     }
 
     @Test
     public void testSkipped() throws Exception {
-        writeReadCheck(new RunResult(3, 2, 1, 0, null, true));
+        writeReadCheck(new RunResult(3, 2, 1, 0, 50, null, true));
     }
 
     @Test
     public void testAppendSerialization() throws Exception {
         RunResult simpleAggregate = getSimpleAggregate();
-        RunResult additional = new RunResult(2, 1, 2, 2, "msg " + ((char) 0x0E01), true);
+        RunResult additional = new RunResult(2, 1, 2, 2, 500, "msg " + ((char) 0x0E01), true);
 
         File summary = SureFireFileManager.createTempFile("failsafe", "test");
         FailsafeSummaryXmlUtils.writeSummary(simpleAggregate, summary, false);
@@ -88,6 +88,8 @@ public class RunResultTest {
 
         assertThat(expected.getFlakes()).isEqualTo(2);
 
+        assertThat(expected.getElapsed()).isEqualTo(500);
+
         assertThat(expected.getFailure()).isEqualTo("msg " + ((char) 0x0E01));
 
         assertThat(expected.isTimeout()).isTrue();
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
index 762e1348c..66f205a94 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -263,7 +263,7 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
         }
 
         // Update globalStatistics by iterating through mergedTestHistoryResult
-        int completedCount = 0, skipped = 0;
+        int completedCount = 0, skipped = 0, elapsed = -1;
 
         for (Map.Entry<String, List<TestMethodStats>> entry : mergedTestHistoryResult.entrySet()) {
             List<TestMethodStats> testMethodStats = entry.getValue();
@@ -273,6 +273,12 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
             List<ReportEntryType> resultTypes = new ArrayList<>();
             for (TestMethodStats methodStats : testMethodStats) {
                 resultTypes.add(methodStats.getResultType());
+                if (methodStats.getElapsed() != null) {
+                    if (elapsed == -1) {
+                        elapsed = 0;
+                    }
+                    elapsed += methodStats.getElapsed();
+                }
             }
 
             switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) {
@@ -303,7 +309,7 @@ public class DefaultReporterFactory implements ReporterFactory, ReportsMerger {
             }
         }
 
-        globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size());
+        globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size(), (elapsed != -1 ? elapsed : null));
     }
 
     /**
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
index d3abb53a7..6033509c9 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java
@@ -34,10 +34,13 @@ public class TestMethodStats {
 
     private final StackTraceWriter stackTraceWriter;
 
-    public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter) {
+    private final Integer elapsed;
+
+    public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter, Integer elapsed) {
         this.testClassMethodName = testClassMethodName;
         this.resultType = resultType;
         this.stackTraceWriter = stackTraceWriter;
+        this.elapsed = elapsed;
     }
 
     public String getTestClassMethodName() {
@@ -51,4 +54,8 @@ public class TestMethodStats {
     public StackTraceWriter getStackTraceWriter() {
         return stackTraceWriter;
     }
+
+    public Integer getElapsed() {
+        return elapsed;
+    }
 }
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
index 5e90fb055..391bc0d94 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java
@@ -288,7 +288,8 @@ public class TestSetRunListener implements TestReportListener<TestOutputReportEn
             TestMethodStats methodStats = new TestMethodStats(
                     reportEntry.getClassMethodName(),
                     reportEntry.getReportEntryType(),
-                    reportEntry.getStackTraceWriter());
+                    reportEntry.getStackTraceWriter(),
+                    reportEntry.getElapsed());
             testMethodStats.add(methodStats);
         }
     }
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
index 00abbf248..6436b8892 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java
@@ -18,12 +18,25 @@
  */
 package org.apache.maven.surefire.report;
 
+import java.text.MessageFormat;
+import java.util.Locale;
+
 import org.apache.maven.surefire.api.suite.RunResult;
 
 /**
  * @author Kristian Rosenvold
  */
 public final class RunStatistics {
+    private static final float ONE_SECOND = 1000.0f;
+
+    /*
+     * Rationale: The idea is to always display four digits for visually consistent output
+     * Important: Keep in sync with maven-surefire-report-plugin/src/main/resources/surefire-report.properties
+     */
+    private final MessageFormat elapsedTimeFormat = new MessageFormat(
+            "{0,choice,0#0|0.0<{0,number,0.000}|10#{0,number,0.00}|100#{0,number,0.0}|1000#{0,number,0}} s",
+            Locale.ROOT);
+
     private int completedCount;
 
     private int errors;
@@ -34,6 +47,8 @@ public final class RunStatistics {
 
     private int flakes;
 
+    private Integer elapsed;
+
     public synchronized int getCompletedCount() {
         return completedCount;
     }
@@ -54,12 +69,17 @@ public final class RunStatistics {
         return flakes;
     }
 
-    public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes) {
+    public synchronized Integer getElapsed() {
+        return elapsed;
+    }
+
+    public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) {
         this.completedCount = completedCount;
         this.errors = errors;
         this.failures = failures;
         this.skipped = skipped;
         this.flakes = flakes;
+        this.elapsed = elapsed;
     }
 
     public synchronized RunResult getRunResult() {
@@ -72,6 +92,7 @@ public final class RunStatistics {
         if (flakes > 0) {
             summary += ", Flakes: " + flakes;
         }
+        summary += ", Time elapsed: " + (elapsed != null ? elapsedTimeFormat.format(new Object[] {elapsed / ONE_SECOND}): "(unknown)");
         return summary;
     }
 }
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
index 9d4c9f924..11e5e59c1 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -72,6 +72,7 @@ public class DefaultReporterFactoryTest extends TestCase {
 
     private static final String ERROR = "error";
 
+    @SuppressWarnings("checkstyle:MagicNumber")
     public void testMergeTestHistoryResult() throws Exception {
         MessageUtils.setColorEnabled(false);
         File target = new File(System.getProperty("user.dir"), "target");
@@ -100,26 +101,28 @@ public class DefaultReporterFactoryTest extends TestCase {
 
         // First run, four tests failed and one passed
         Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
-        firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
-        firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
+        //CHECKSTYLE:OFF
+        firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
+        firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
         firstRunStats.add(
-                new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
+                new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
         firstRunStats.add(
-                new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
-        firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
+                new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
+        firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null, 50));
 
         // Second run, two tests passed
         Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
         secondRunStats.add(
-                new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL)));
-        secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null));
-        secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
-        secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null));
+                new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50));
+        secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null, 25));
+        secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), 50));
+        secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null, 25));
 
         // Third run, another test passed
         Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
-        thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null));
-        thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
+        thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null, 10));
+        thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null));
+        //CHECKSTYLE:ON
 
         TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
         TestSetRunListener secondRunListener = mock(TestSetRunListener.class);
@@ -402,7 +405,7 @@ public class DefaultReporterFactoryTest extends TestCase {
         assertEquals(0, statistics.getErrors());
         assertEquals(0, statistics.getSkipped());
         assertEquals(0, statistics.getFlakes());
-        assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary());
+        assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", statistics.getSummary());
         assertEquals(0, statistics.getCompletedCount());
 
         List<String> messages = reporter.getMessages();
@@ -413,7 +416,7 @@ public class DefaultReporterFactoryTest extends TestCase {
         assertEquals("", messages.get(4));
         assertEquals("Results:", messages.get(5));
         assertEquals("", messages.get(6));
-        assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7));
+        assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", messages.get(7));
         assertEquals("", messages.get(8));
         assertEquals(9, messages.size());
     }
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
index a3206e00a..eb6861966 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java
@@ -23,10 +23,11 @@ import junit.framework.TestCase;
 /**
  *
  */
+@SuppressWarnings("checkstyle:MagicNumber")
 public class RunStatisticsTest extends TestCase {
     public void testSetRunStatistics() {
         RunStatistics statistics = new RunStatistics();
-        statistics.set(10, 5, 2, 1, 2);
+        statistics.set(10, 5, 2, 1, 2, 5500);
         assertEquals(10, statistics.getCompletedCount());
         assertEquals(5, statistics.getErrors());
         assertEquals(2, statistics.getFailures());
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
index ceb182aee..fee341212 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java
@@ -40,6 +40,8 @@ public class RunResult {
 
     private final int flakes;
 
+    private final Integer elapsed;
+
     private final String failure;
 
     private final boolean timeout;
@@ -58,37 +60,44 @@ public class RunResult {
         return errorCode(accumulatedAtTimeout, getStackTrace(cause), accumulatedAtTimeout.isTimeout());
     }
 
+    // TODO What happened to flakes?
     private static RunResult errorCode(RunResult other, String failure, boolean timeout) {
         return new RunResult(
                 other.getCompletedCount(),
                 other.getErrors(),
                 other.getFailures(),
                 other.getSkipped(),
+                other.getElapsed(),
                 failure,
                 timeout);
     }
 
     public RunResult(int completedCount, int errors, int failures, int skipped) {
-        this(completedCount, errors, failures, skipped, null, false);
+        this(completedCount, errors, failures, skipped, null, null, false);
     }
 
     public RunResult(int completedCount, int errors, int failures, int skipped, int flakes) {
-        this(completedCount, errors, failures, skipped, flakes, null, false);
+        this(completedCount, errors, failures, skipped, flakes, null, null, false);
+    }
+
+    public RunResult(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) {
+        this(completedCount, errors, failures, skipped, flakes, elapsed, null, false);
     }
 
-    public RunResult(int completedCount, int errors, int failures, int skipped, String failure, boolean timeout) {
-        this(completedCount, errors, failures, skipped, 0, failure, timeout);
+    public RunResult(int completedCount, int errors, int failures, int skipped, Integer elapsed, String failure, boolean timeout) {
+        this(completedCount, errors, failures, skipped, 0, elapsed, failure, timeout);
     }
 
     public RunResult(
-            int completedCount, int errors, int failures, int skipped, int flakes, String failure, boolean timeout) {
+            int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed, String failure, boolean timeout) {
         this.completedCount = completedCount;
         this.errors = errors;
         this.failures = failures;
         this.skipped = skipped;
+        this.flakes = flakes;
+        this.elapsed = elapsed;
         this.failure = failure;
         this.timeout = timeout;
-        this.flakes = flakes;
     }
 
     private static String getStackTrace(Exception e) {
@@ -122,6 +131,10 @@ public class RunResult {
         return skipped;
     }
 
+    public Integer getElapsed() {
+        return elapsed;
+    }
+
     public Integer getFailsafeCode() // Only used for compatibility reasons.
             {
         if (completedCount == 0) {
@@ -167,13 +180,24 @@ public class RunResult {
         int ign = getSkipped() + other.getSkipped();
         int err = getErrors() + other.getErrors();
         int flakes = getFlakes() + other.getFlakes();
-        return new RunResult(completed, err, fail, ign, flakes, failureMessage, timeout);
+        Integer elapsed;
+        if (getElapsed() == null) {
+            elapsed = other.getElapsed();
+        } else {
+            if (other.getElapsed() == null) {
+                elapsed = getElapsed();
+            } else {
+                elapsed = getElapsed() + other.getElapsed();
+            }
+        }
+        return new RunResult(completed, err, fail, ign, flakes, elapsed, failureMessage, timeout);
     }
 
     public static RunResult noTestsRun() {
         return new RunResult(0, 0, 0, 0);
     }
 
+    // TODO What happened to flakes?
     @Override
     @SuppressWarnings("RedundantIfStatement")
     public boolean equals(Object o) {
@@ -208,6 +232,7 @@ public class RunResult {
         return true;
     }
 
+    // TODO What happened to flakes?
     @Override
     public int hashCode() {
         int result = completedCount;