You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by ti...@apache.org on 2021/01/14 23:14:56 UTC
[maven-surefire] branch comm updated: performance = buffering and
closing the stream of *.bin files
This is an automated email from the ASF dual-hosted git repository.
tibordigana pushed a commit to branch comm
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/comm by this push:
new 784142e performance = buffering and closing the stream of *.bin files
784142e is described below
commit 784142e4eb023b9364d4a65676f4defc9115d0bd
Author: tibordigana <ti...@gmail.com>
AuthorDate: Fri Jan 15 00:14:46 2021 +0100
performance = buffering and closing the stream of *.bin files
---
.../plugin/surefire/booterclient/ForkStarter.java | 11 ++--
.../output/InPluginProcessDumpSingleton.java | 1 +
.../surefire/extensions/EventConsumerThread.java | 7 ++-
.../apache/maven/surefire/stream/EventDecoder.java | 70 ++++++++++++++++++++++
.../surefire/api/booter/DumpErrorSingleton.java | 2 +-
.../surefire/api/stream/AbstractStreamDecoder.java | 2 +-
.../surefire/api/util/internal/DumpFileUtils.java | 1 +
.../api/stream/AbstractStreamDecoderTest.java | 5 ++
.../maven/surefire/booter/CommandReader.java | 4 +-
.../surefire/booter/spi/CommandChannelDecoder.java | 3 +-
.../surefire/booter/stream/CommandDecoder.java | 14 ++++-
11 files changed, 107 insertions(+), 13 deletions(-)
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
index 55647a1..200d2a8 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
@@ -574,7 +574,8 @@ public class ForkStarter
File dumpLogDir = replaceForkThreadsInPath( startupReportConfiguration.getReportsDirectory(), forkNumber );
try
{
- ForkNodeArguments forkNodeArguments = new ForkedNodeArg( forkNumber, dumpLogDir, randomUUID().toString() );
+ ForkNodeArguments forkNodeArguments =
+ new ForkedNodeArg( forkConfiguration.isDebug(), forkNumber, dumpLogDir, randomUUID().toString() );
forkChannel = forkNodeFactory.createForkChannel( forkNodeArguments );
closer.addCloseable( forkChannel );
tempDir = forkConfiguration.getTempDirectory().getCanonicalPath();
@@ -884,9 +885,11 @@ public class ForkStarter
private final int forkChannelId;
private final File dumpLogDir;
private final String sessionId;
+ private final boolean debug;
- ForkedNodeArg( int forkChannelId, File dumpLogDir, String sessionId )
+ ForkedNodeArg( boolean debug, int forkChannelId, File dumpLogDir, String sessionId )
{
+ this.debug = debug;
this.forkChannelId = forkChannelId;
this.dumpLogDir = dumpLogDir;
this.sessionId = sessionId;
@@ -936,8 +939,8 @@ public class ForkStarter
@Override
public File getEventStreamBinaryFile()
{
- return InPluginProcessDumpSingleton.getSingleton()
- .getEventStreamBinaryFile( dumpLogDir, forkChannelId );
+ return debug ? InPluginProcessDumpSingleton.getSingleton()
+ .getEventStreamBinaryFile( dumpLogDir, forkChannelId ) : null;
}
@Override
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/InPluginProcessDumpSingleton.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/InPluginProcessDumpSingleton.java
index aa4fa3c..05c4235 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/InPluginProcessDumpSingleton.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/InPluginProcessDumpSingleton.java
@@ -85,6 +85,7 @@ public final class InPluginProcessDumpSingleton
public File getEventStreamBinaryFile( File reportsDirectory, int jvmRun )
{
+ reportsDirectory.mkdirs();
return new File( reportsDirectory, format( EVENTS_BINARY_DUMP_FILENAME_FORMATTER, jvmRun ) );
}
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/EventConsumerThread.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/EventConsumerThread.java
index 55fb4d0..1371dbb 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/EventConsumerThread.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/EventConsumerThread.java
@@ -61,13 +61,14 @@ public class EventConsumerThread extends CloseableDaemonThread
@Override
public void run()
{
- Memento memento = decoder.new Memento();
try ( ReadableByteChannel stream = channel;
- CountdownCloseable c = countdownCloseable; )
+ CountdownCloseable c = countdownCloseable;
+ EventDecoder eventDecoder = decoder )
{
+ Memento memento = eventDecoder.new Memento();
do
{
- Event event = decoder.decode( memento );
+ Event event = eventDecoder.decode( memento );
if ( event != null && !disabled )
{
eventHandler.handleEvent( event );
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/stream/EventDecoder.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/stream/EventDecoder.java
index 1d7a2a4..3698511 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/stream/EventDecoder.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/stream/EventDecoder.java
@@ -51,12 +51,19 @@ import org.apache.maven.surefire.api.stream.AbstractStreamDecoder;
import org.apache.maven.surefire.api.stream.SegmentType;
import javax.annotation.Nonnull;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.channels.ReadableByteChannel;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
import static org.apache.maven.surefire.api.booter.Constants.MAGIC_NUMBER_FOR_EVENTS_BYTES;
import static org.apache.maven.surefire.api.report.CategorizedReportEntry.reportEntry;
@@ -71,6 +78,7 @@ import static org.apache.maven.surefire.api.stream.SegmentType.STRING_ENCODING;
*/
public class EventDecoder extends AbstractStreamDecoder<Event, ForkedProcessEventType, SegmentType>
{
+ private static final int DEBUG_SINK_BUFFER_SIZE = 64 * 1024;
// due to have fast and thread-safe Map
private static final Map<Segment, ForkedProcessEventType> EVENT_TYPES = segmentsToEvents();
private static final Map<Segment, RunMode> RUN_MODES = segmentsToRunModes();
@@ -126,10 +134,13 @@ public class EventDecoder extends AbstractStreamDecoder<Event, ForkedProcessEven
private static final int NO_POSITION = -1;
+ private final OutputStream debugSink;
+
public EventDecoder( @Nonnull ReadableByteChannel channel,
@Nonnull ForkNodeArguments arguments )
{
super( channel, arguments, EVENT_TYPES );
+ debugSink = newDebugSink( arguments );
}
@Override
@@ -395,4 +406,63 @@ public class EventDecoder extends AbstractStreamDecoder<Event, ForkedProcessEven
}
return runModes;
}
+
+ @Override
+ protected void debugStream( byte[] array, int position, int remaining )
+ {
+ if ( debugSink == null )
+ {
+ return;
+ }
+
+ try
+ {
+ debugSink.write( array, position, remaining );
+ debugSink.flush();
+ }
+ catch ( IOException e )
+ {
+ // logger file was deleted
+ // System.out is already used by the stream in this decoder
+ }
+ }
+
+ private OutputStream newDebugSink( ForkNodeArguments arguments )
+ {
+ final File sink = arguments.getEventStreamBinaryFile();
+ if ( sink == null )
+ {
+ return null;
+ }
+
+ try
+ {
+ OutputStream fos = new FileOutputStream( sink, true );
+ final OutputStream os = new BufferedOutputStream( fos, DEBUG_SINK_BUFFER_SIZE );
+ Runtime.getRuntime().addShutdownHook( new Thread( new FutureTask<>( new Callable<Void>()
+ {
+ @Override
+ public Void call() throws Exception
+ {
+ os.close();
+ return null;
+ }
+ } ) ) );
+ return os;
+ }
+ catch ( FileNotFoundException e )
+ {
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ // do NOT close the channel, it's std/out.
+ if ( debugSink != null )
+ {
+ debugSink.close();
+ }
+ }
}
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/DumpErrorSingleton.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/DumpErrorSingleton.java
index 381f852..791dae5 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/DumpErrorSingleton.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/DumpErrorSingleton.java
@@ -57,7 +57,7 @@ public final class DumpErrorSingleton
dumpStreamFile = createDumpStreamFile( reportsDir, dumpFileName );
String fileNameWithoutExtension =
dumpFileName.contains( "." ) ? dumpFileName.substring( 0, dumpFileName.lastIndexOf( '.' ) ) : dumpFileName;
- binaryDumpStreamFile = createDumpStreamFile( reportsDir, fileNameWithoutExtension + "-commands.bin" );
+ binaryDumpStreamFile = newDumpFile( reportsDir, fileNameWithoutExtension + "-commands.bin" );
}
public synchronized File dumpException( Throwable t, String msg )
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoder.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoder.java
index e989d12..862820c 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoder.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoder.java
@@ -53,7 +53,7 @@ import static org.apache.maven.surefire.shared.lang3.StringUtils.isBlank;
* @param <MT> enum describing the meaning of the message
* @param <ST> enum for segment type
*/
-public abstract class AbstractStreamDecoder<M, MT extends Enum<MT>, ST extends Enum<ST>>
+public abstract class AbstractStreamDecoder<M, MT extends Enum<MT>, ST extends Enum<ST>> implements AutoCloseable
{
public static final int BUFFER_SIZE = 1024;
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/internal/DumpFileUtils.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/internal/DumpFileUtils.java
index bac68be..b552aa2 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/internal/DumpFileUtils.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/internal/DumpFileUtils.java
@@ -52,6 +52,7 @@ public final class DumpFileUtils
*/
public static synchronized File newDumpFile( File reportsDir, String dumpFileName )
{
+ reportsDir.mkdirs();
return new File( reportsDir, dumpFileName );
}
diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoderTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoderTest.java
index 3f4c0bd..5ab7ae0 100644
--- a/surefire-api/src/test/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoderTest.java
+++ b/surefire-api/src/test/java/org/apache/maven/surefire/api/stream/AbstractStreamDecoderTest.java
@@ -683,5 +683,10 @@ public class AbstractStreamDecoderTest
{
return null;
}
+
+ @Override
+ public void close() throws Exception
+ {
+ }
}
}
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/CommandReader.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/CommandReader.java
index ebe72a9..6599433 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/CommandReader.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/CommandReader.java
@@ -336,11 +336,11 @@ public final class CommandReader implements CommandChainReader
{
CommandReader.this.startMonitor.countDown();
boolean isTestSetFinished = false;
- try
+ try ( MasterProcessChannelDecoder commandReader = CommandReader.this.decoder )
{
while ( CommandReader.this.state.get() == RUNNABLE )
{
- Command command = CommandReader.this.decoder.decode();
+ Command command = commandReader.decode();
switch ( command.getCommandType() )
{
case RUN_CLASS:
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/spi/CommandChannelDecoder.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/spi/CommandChannelDecoder.java
index 0e46be3..85bda73 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/spi/CommandChannelDecoder.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/spi/CommandChannelDecoder.java
@@ -79,7 +79,8 @@ public class CommandChannelDecoder implements MasterProcessChannelDecoder
}
@Override
- public void close()
+ public void close() throws IOException
{
+ decoder.close();
}
}
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/stream/CommandDecoder.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/stream/CommandDecoder.java
index 873136f..b660147 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/stream/CommandDecoder.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/stream/CommandDecoder.java
@@ -29,6 +29,7 @@ import org.apache.maven.surefire.api.stream.MalformedChannelException;
import org.apache.maven.surefire.api.stream.SegmentType;
import javax.annotation.Nonnull;
+import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -57,6 +58,7 @@ import static org.apache.maven.surefire.api.stream.SegmentType.STRING_ENCODING;
*/
public class CommandDecoder extends AbstractStreamDecoder<Command, MasterProcessCommand, SegmentType>
{
+ private static final int DEBUG_SINK_BUFFER_SIZE = 64 * 1024;
private static final int NO_POSITION = -1;
private static final SegmentType[] COMMAND_WITHOUT_DATA = new SegmentType[] {
@@ -244,7 +246,8 @@ public class CommandDecoder extends AbstractStreamDecoder<Command, MasterProcess
try
{
- final OutputStream os = new FileOutputStream( sink, true );
+ OutputStream fos = new FileOutputStream( sink, true );
+ final OutputStream os = new BufferedOutputStream( fos, DEBUG_SINK_BUFFER_SIZE );
Runtime.getRuntime().addShutdownHook( new Thread( new FutureTask<>( new Callable<Void>()
{
@Override
@@ -261,4 +264,13 @@ public class CommandDecoder extends AbstractStreamDecoder<Command, MasterProcess
return null;
}
}
+
+ @Override
+ public void close() throws IOException
+ {
+ if ( debugSink != null )
+ {
+ debugSink.close();
+ }
+ }
}