You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ti...@apache.org on 2022/01/28 00:04:28 UTC
[maven-surefire] 01/01: [SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an assumption
This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch SUREFIRE-1556
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
commit 5614343401a54dcdef8eb239243987d8bf97a2f6
Author: Tibor Digaňa <ti...@apache.org>
AuthorDate: Fri Jan 28 01:04:12 2022 +0100
[SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an assumption
---
.../surefire/report/StatelessXmlReporter.java | 60 ++++++++----
.../surefire/report/StatelessXmlReporterTest.java | 106 +++++++++++++++++----
2 files changed, 130 insertions(+), 36 deletions(-)
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
index 984bd94..485e5a2 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
@@ -25,13 +25,7 @@ import org.apache.maven.surefire.shared.utils.xml.XMLWriter;
import org.apache.maven.surefire.extensions.StatelessReportEventListener;
import org.apache.maven.surefire.api.report.SafeThrowable;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
+import java.io.*;
import java.util.ArrayList;
import java.util.Deque;
import java.util.LinkedHashMap;
@@ -44,6 +38,7 @@ import java.util.concurrent.ConcurrentLinkedDeque;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars;
+import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank;
@@ -85,6 +80,10 @@ import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank;
public class StatelessXmlReporter
implements StatelessReportEventListener<WrappedReportEntry, TestSetStats>
{
+ private static final String XML_INDENT = " ";
+
+ private static final String XML_NL = "\n";
+
private final File reportsDirectory;
private final String reportNameSuffix;
@@ -139,8 +138,7 @@ public class StatelessXmlReporter
try ( OutputStream outputStream = getOutputStream( testSetReportEntry );
OutputStreamWriter fw = getWriter( outputStream ) )
{
- XMLWriter ppw = new PrettyPrintXMLWriter( fw );
- ppw.setEncoding( UTF_8.name() );
+ XMLWriter ppw = new PrettyPrintXMLWriter( new PrintWriter( fw ), XML_INDENT, XML_NL, UTF_8.name(), null );
createTestSuiteElement( ppw, testSetReportEntry, testSetStats ); // TestSuite
@@ -264,6 +262,14 @@ public class StatelessXmlReporter
singleRunEntry.getReportEntryType().getXmlTag(), false );
createOutErrElements( fw, ppw, singleRunEntry, outputStream );
}
+ else if ( singleRunEntry.getReportEntryType() == SKIPPED )
+ {
+ // The version 3.1.0 should produce a new XSD schema with version 3.1.0, see SUREFIRE-1986,
+ // and the XSD schema should add a new element "rerunSkipped"
+ // then ReportEntryType should update the enum to SKIPPED( "skipped", "", "rerunSkipped" ).
+ // The teams should be notified - Jenkins reports.
+ addCommentElementTestCase( "a skipped test execution in re-run phase", fw, ppw, outputStream );
+ }
else
{
getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
@@ -347,6 +353,7 @@ public class StatelessXmlReporter
{
File reportFile = getReportFile( testSetReportEntry );
File reportDir = reportFile.getParentFile();
+ //noinspection ResultOfMethodCallIgnored
reportFile.delete();
//noinspection ResultOfMethodCallIgnored
reportDir.mkdirs();
@@ -572,6 +579,23 @@ public class StatelessXmlReporter
}
}
+ private static void addCommentElementTestCase(String comment, OutputStreamWriter outputStreamWriter,
+ XMLWriter xmlWriter, OutputStream fw )
+ throws IOException
+ {
+ xmlWriter.writeText( "" ); // Cheat sax to emit element
+ outputStreamWriter.flush();
+ fw.write( XML_NL.getBytes( UTF_8 ) );
+ fw.write( XML_INDENT.getBytes( UTF_8 ) );
+ fw.write( XML_INDENT.getBytes( UTF_8 ) );
+ fw.write( ByteConstantsHolder.COMMENT_START );
+ fw.write( comment.getBytes( UTF_8 ) );
+ fw.write( ByteConstantsHolder.COMMENT_END );
+ fw.write( XML_NL.getBytes( UTF_8 ) );
+ fw.write( XML_INDENT.getBytes( UTF_8 ) );
+ fw.flush();
+ }
+
private static final class EncodingOutputStream
extends FilterOutputStream
{
@@ -679,20 +703,16 @@ public class StatelessXmlReporter
private static final class ByteConstantsHolder
{
- private static final byte[] CDATA_START_BYTES;
+ private static final byte[] CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );
- private static final byte[] CDATA_END_BYTES;
+ private static final byte[] CDATA_END_BYTES = "]]>".getBytes( UTF_8 );
- private static final byte[] CDATA_ESCAPE_STRING_BYTES;
+ private static final byte[] CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );
- private static final byte[] AMP_BYTES;
+ private static final byte[] AMP_BYTES = "&#".getBytes( UTF_8 );
- static
- {
- CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );
- CDATA_END_BYTES = "]]>".getBytes( UTF_8 );
- CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );
- AMP_BYTES = "&#".getBytes( UTF_8 );
- }
+ private static final byte[] COMMENT_START = "<!-- ".getBytes( UTF_8 );
+
+ private static final byte[] COMMENT_END = " --> ".getBytes( UTF_8 );
}
}
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
index 35fad19..00c8077 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
@@ -24,7 +24,6 @@ import org.apache.maven.plugin.surefire.booterclient.output.DeserializedStacktra
import org.apache.maven.surefire.api.report.ReportEntry;
import org.apache.maven.surefire.api.report.SimpleReportEntry;
import org.apache.maven.surefire.api.report.StackTraceWriter;
-import org.apache.maven.surefire.shared.utils.StringUtils;
import org.apache.maven.surefire.shared.utils.xml.Xpp3Dom;
import org.apache.maven.surefire.shared.utils.xml.Xpp3DomBuilder;
@@ -36,6 +35,7 @@ import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.nio.Buffer;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Deque;
import java.util.HashMap;
@@ -43,7 +43,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.readAllLines;
+import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
+import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
+import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
+import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.api.util.internal.ObjectUtils.systemProps;
+import static org.apache.maven.surefire.shared.utils.StringUtils.isEmpty;
import static org.fest.assertions.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -107,7 +113,7 @@ public class StatelessXmlReporterTest
reporter.cleanTestHistoryMap();
ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, getClass().getName(), null, 12 );
- WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS,
+ WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, SUCCESS,
12, null, null, systemProps() );
stats.testSucceeded( testSetReportEntry );
reporter.testSetCompleted( testSetReportEntry, stats );
@@ -123,7 +129,7 @@ public class StatelessXmlReporterTest
{
ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 );
WrappedReportEntry testSetReportEntry =
- new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, 12, null, null, systemProps() );
+ new WrappedReportEntry( reportEntry, SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );
stats.testSucceeded( testSetReportEntry );
@@ -149,7 +155,7 @@ public class StatelessXmlReporterTest
stdErr.write( stdErrPrefix + "?&-&£\u0020\u0000\u001F", false );
WrappedReportEntry t2 =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
- stackTraceWriter, 13 ), ReportEntryType.ERROR, 13, stdOut, stdErr );
+ stackTraceWriter, 13 ), ERROR, 13, stdOut, stdErr );
stats.testSucceeded( t2 );
StatelessXmlReporter reporter = new StatelessXmlReporter( reportDir, null, false, 0,
@@ -163,8 +169,8 @@ public class StatelessXmlReporterTest
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+ assertFalse( isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( isEmpty( child.getAttribute( "name" ) ) );
Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom tca = testcase[0];
@@ -191,7 +197,7 @@ public class StatelessXmlReporterTest
{
WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 ),
- ReportEntryType.SUCCESS, 12, null, null, systemProps() );
+ SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );
stats.testSucceeded( testSetReportEntry );
@@ -207,27 +213,27 @@ public class StatelessXmlReporterTest
WrappedReportEntry testTwoFirstError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
- stackTraceWriterOne, 5 ), ReportEntryType.ERROR, 5, createStdOutput( firstRunOut ),
+ stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );
WrappedReportEntry testTwoSecondError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
- stackTraceWriterTwo, 13 ), ReportEntryType.ERROR, 13, createStdOutput( secondRunOut ),
+ stackTraceWriterTwo, 13 ), ERROR, 13, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );
WrappedReportEntry testThreeFirstRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
- stackTraceWriterOne, 13 ), ReportEntryType.FAILURE, 13, createStdOutput( firstRunOut ),
+ stackTraceWriterOne, 13 ), FAILURE, 13, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );
WrappedReportEntry testThreeSecondRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
- stackTraceWriterTwo, 2 ), ReportEntryType.SUCCESS, 2, createStdOutput( secondRunOut ),
+ stackTraceWriterTwo, 2 ), SUCCESS, 2, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );
- stats.testSucceeded( testTwoFirstError );
- stats.testSucceeded( testThreeFirstRun );
- rerunStats.testSucceeded( testTwoSecondError );
+ stats.testError( testTwoFirstError );
+ stats.testFailure( testThreeFirstRun );
+ rerunStats.testError( testTwoSecondError );
rerunStats.testSucceeded( testThreeSecondRun );
StatelessXmlReporter reporter =
@@ -236,6 +242,7 @@ public class StatelessXmlReporterTest
reporter.testSetCompleted( testSetReportEntry, stats );
reporter.testSetCompleted( testSetReportEntry, rerunStats );
+ System.out.println(new String(Files.readAllBytes(expectedReportFile.toPath())));
FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
@@ -245,8 +252,8 @@ public class StatelessXmlReporterTest
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
- assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
+ assertFalse( isEmpty( child.getAttribute( "value" ) ) );
+ assertFalse( isEmpty( child.getAttribute( "name" ) ) );
Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom testCaseOne = testcase[0];
@@ -290,6 +297,73 @@ public class StatelessXmlReporterTest
assertNull( testCaseThree.getChild( "system-err" ) );
}
+ public void testOutputRerunFlakyAssumption()
+ throws IOException
+ {
+ expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );
+
+ StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed",
+ "fail at foo" );
+
+ StackTraceWriter stackTraceWriterTwo =
+ new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" );
+
+ String firstRunOut = "first run out";
+ String firstRunErr = "first run err";
+ String secondRunOut = "second run out";
+ String secondRunErr = "second run err";
+
+ WrappedReportEntry testTwoFirstError =
+ new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
+ stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ stats.testSucceeded( testTwoFirstError );
+
+ WrappedReportEntry testTwoSecondError =
+ new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
+ stackTraceWriterTwo, 13 ), SKIPPED, 13, createStdOutput( secondRunOut ),
+ createStdOutput( secondRunErr ) );
+
+ rerunStats.testSucceeded( testTwoSecondError );
+
+ StatelessXmlReporter reporter =
+ new StatelessXmlReporter( reportDir, null, false, 1,
+ new HashMap<>(), XSD, "3.0", false, false, false, false );
+
+ WrappedReportEntry testSetReportEntry =
+ new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, null, null,
+ stackTraceWriterOne, 5 ), ERROR, 20, createStdOutput( firstRunOut ),
+ createStdOutput( firstRunErr ) );
+
+ reporter.testSetCompleted( testSetReportEntry, stats );
+ reporter.testSetCompleted( testSetReportEntry, rerunStats );
+
+ FileInputStream fileInputStream = new FileInputStream( expectedReportFile );
+
+ Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, UTF_8 ) );
+ assertEquals( "testsuite", testSuite.getName() );
+ assertEquals( "0.02", testSuite.getAttribute( "time" ) );
+
+ Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
+ assertEquals( 1, testcase.length );
+ Xpp3Dom testCaseOne = testcase[0];
+ assertEquals( getClass().getName(), testCaseOne.getAttribute( "classname" ) );
+ assertEquals( TEST_TWO, testCaseOne.getAttribute( "name" ) );
+ assertEquals( "0.005", testCaseOne.getAttribute( "time" ) );
+
+ Xpp3Dom[] testCaseElements = testCaseOne.getChildren();
+ assertEquals( 3, testCaseElements.length );
+ assertEquals( "error", testCaseElements[0].getName() );
+ assertEquals( "system-out", testCaseElements[1].getName() );
+ assertEquals( "system-err", testCaseElements[2].getName() );
+ long linesWithComments = readAllLines( expectedReportFile.toPath(), UTF_8 )
+ .stream()
+ .filter( line -> line.contains( "<!-- a skipped test execution in re-run phase -->" ) )
+ .count();
+ assertEquals( 1, linesWithComments );
+ }
+
public void testNoWritesOnDeferredFile() throws Exception
{
Utf8RecodingDeferredFileOutputStream out = new Utf8RecodingDeferredFileOutputStream( "test" );