You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rp...@apache.org on 2016/04/13 18:23:18 UTC
[1/2] logging-log4j2 git commit: LOG4J2-1274 added
StringBuilderEncoder
Repository: logging-log4j2
Updated Branches:
refs/heads/master 90adca7a5 -> 0a06f8f0d
LOG4J2-1274 added StringBuilderEncoder
- client code now uses StringBuilderEncoder (or Encoder<StringBuilder>) instead of TextEncoderHelper
- LockingStringBuilderEncoder is a version that synchronizes on ByteBufferDestination during the whole encoding process
- StringBuilderEncoder has a ThreadLocal temp buffer used for the encoding, and only synchronizes on ByteBufferDestination during the final copy
- updated tests and benchmarks
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/23cd33fc
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/23cd33fc
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/23cd33fc
Branch: refs/heads/master
Commit: 23cd33fcb253b318e4dab532e0ce68db274854ef
Parents: 90adca7
Author: rpopma <rp...@apache.org>
Authored: Thu Apr 14 01:17:42 2016 +0900
Committer: rpopma <rp...@apache.org>
Committed: Thu Apr 14 01:17:42 2016 +0900
----------------------------------------------------------------------
.../log4j/core/layout/AbstractStringLayout.java | 21 +-
.../logging/log4j/core/layout/GelfLayout.java | 11 +-
.../layout/LockingStringBuilderEncoder.java | 52 ++++
.../log4j/core/layout/PatternLayout.java | 5 +-
.../log4j/core/layout/StringBuilderEncoder.java | 111 ++++++++
.../log4j/core/layout/TextEncoderHelper.java | 146 +++++-----
.../core/layout/StringBuilderEncoderTest.java | 277 +++++++++++++++++++
.../core/layout/TextEncoderHelperTest.java | 263 ------------------
.../perf/jmh/TextEncoderHelperBenchmark.java | 97 ++++---
.../logging/log4j/perf/nogc/NoGcLayout.java | 11 +-
10 files changed, 596 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
index 757e9da..6694943 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
@@ -59,7 +59,7 @@ public abstract class AbstractStringLayout extends AbstractLayout<String> implem
private static final ThreadLocal<StringBuilder> threadLocal = new ThreadLocal<>();
- private TextEncoderHelper textEncoderHelper;
+ private Encoder<StringBuilder> textEncoder;
/**
* Returns a {@code StringBuilder} that this Layout implementation can use to write the formatted log event to.
@@ -120,7 +120,7 @@ public abstract class AbstractStringLayout extends AbstractLayout<String> implem
this.charsetName = this.charset.name();
useCustomEncoding = isPreJava8()
&& (StandardCharsets.ISO_8859_1.equals(aCharset) || StandardCharsets.US_ASCII.equals(aCharset));
- textEncoderHelper = Constants.ENABLE_DIRECT_ENCODERS ? new TextEncoderHelper(charset) : null;
+ textEncoder = Constants.ENABLE_DIRECT_ENCODERS ? new StringBuilderEncoder(charset) : null;
}
/**
@@ -131,7 +131,8 @@ public abstract class AbstractStringLayout extends AbstractLayout<String> implem
* @param headerSerializer the header bytes serializer
* @param footerSerializer the footer bytes serializer
*/
- protected AbstractStringLayout(final Configuration config, final Charset aCharset, final Serializer headerSerializer, final Serializer footerSerializer) {
+ protected AbstractStringLayout(final Configuration config, final Charset aCharset,
+ final Serializer headerSerializer, final Serializer footerSerializer) {
super(config, null, null);
this.headerSerializer = headerSerializer;
this.footerSerializer = footerSerializer;
@@ -139,19 +140,19 @@ public abstract class AbstractStringLayout extends AbstractLayout<String> implem
this.charsetName = this.charset.name();
useCustomEncoding = isPreJava8()
&& (StandardCharsets.ISO_8859_1.equals(aCharset) || StandardCharsets.US_ASCII.equals(aCharset));
- textEncoderHelper = Constants.ENABLE_DIRECT_ENCODERS ? new TextEncoderHelper(charset) : null;
+ textEncoder = Constants.ENABLE_DIRECT_ENCODERS ? new StringBuilderEncoder(charset) : null;
}
/**
- * Returns a {@code TextEncoderHelper} that this Layout implementation can use for encoding log events.
+ * Returns a {@code Encoder<StringBuilder>} that this Layout implementation can use for encoding log events.
*
- * @return a {@code TextEncoderHelper}
+ * @return a {@code Encoder<StringBuilder>}
*/
- protected TextEncoderHelper getCachedTextEncoderHelper() {
- if (textEncoderHelper == null) {
- textEncoderHelper = new TextEncoderHelper(getCharset());
+ protected Encoder<StringBuilder> getStringBuilderEncoder() {
+ if (textEncoder == null) {
+ textEncoder = new StringBuilderEncoder(getCharset());
}
- return textEncoderHelper;
+ return textEncoder;
}
protected byte[] getBytes(final String s) {
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
index cef4e46..788ce73 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
@@ -25,7 +25,6 @@ import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginElement;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.net.Severity;
-import org.apache.logging.log4j.core.util.Constants;
import org.apache.logging.log4j.core.util.JsonUtils;
import org.apache.logging.log4j.core.util.KeyValuePair;
import org.apache.logging.log4j.message.Message;
@@ -33,7 +32,11 @@ import org.apache.logging.log4j.status.StatusLogger;
import org.apache.logging.log4j.util.StringBuilderFormattable;
import org.apache.logging.log4j.util.Strings;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Map;
@@ -149,8 +152,8 @@ public final class GelfLayout extends AbstractStringLayout {
return;
}
final StringBuilder text = toText(event, getStringBuilder(), true);
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- helper.encodeText(text, destination);
+ final Encoder<StringBuilder> helper = getStringBuilderEncoder();
+ helper.encode(text, destination);
}
private byte[] compress(final byte[] bytes) {
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/LockingStringBuilderEncoder.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/LockingStringBuilderEncoder.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/LockingStringBuilderEncoder.java
new file mode 100644
index 0000000..6224175
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/LockingStringBuilderEncoder.java
@@ -0,0 +1,52 @@
+package org.apache.logging.log4j.core.layout;
+
+import org.apache.logging.log4j.status.StatusLogger;
+
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.util.Objects;
+
+/**
+ * Encoder for StringBuilders that locks on the ByteBufferDestination.
+ */
+public class LockingStringBuilderEncoder implements Encoder<StringBuilder> {
+
+ private final Charset charset;
+ private final CharsetEncoder charsetEncoder;
+ private final CharBuffer cachedCharBuffer;
+
+ public LockingStringBuilderEncoder(final Charset charset) {
+ this(charset, TextEncoderHelper.DEFAULT_CHAR_BUFFER_SIZE);
+ }
+
+ public LockingStringBuilderEncoder(final Charset charset, final int charBufferSize) {
+ this.charset = Objects.requireNonNull(charset, "charset");
+ this.charsetEncoder = charset.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
+ .onUnmappableCharacter(CodingErrorAction.REPLACE);
+ this.cachedCharBuffer = CharBuffer.wrap(new char[charBufferSize]);
+ }
+
+ private CharBuffer getCharBuffer() {
+ return cachedCharBuffer;
+ }
+
+ @Override
+ public void encode(final StringBuilder source, final ByteBufferDestination destination) {
+ synchronized (destination) {
+ try {
+ TextEncoderHelper.encodeText(charsetEncoder, cachedCharBuffer, destination.getByteBuffer(), source,
+ destination);
+ } catch (final Exception ex) {
+ logEncodeTextException(ex, source, destination);
+ TextEncoderHelper.encodeTextFallBack(charset, source, destination);
+ }
+ }
+ }
+
+ private void logEncodeTextException(final Exception ex, final StringBuilder text,
+ final ByteBufferDestination destination) {
+ StatusLogger.getLogger().error("Recovering from LockingStringBuilderEncoder.encode('{}') error", text, ex);
+ }
+}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
index ce04f52..67466be 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
@@ -38,7 +38,6 @@ import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
import org.apache.logging.log4j.core.pattern.PatternFormatter;
import org.apache.logging.log4j.core.pattern.PatternParser;
import org.apache.logging.log4j.core.pattern.RegexReplacement;
-import org.apache.logging.log4j.core.util.Constants;
import org.apache.logging.log4j.util.Strings;
/**
@@ -179,8 +178,8 @@ public final class PatternLayout extends AbstractStringLayout {
return;
}
final StringBuilder text = toText((Serializer2) eventSerializer, event, getStringBuilder());
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- helper.encodeText(text, destination);
+ final Encoder<StringBuilder> encoder = getStringBuilderEncoder();
+ encoder.encode(text, destination);
}
/**
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder.java
new file mode 100644
index 0000000..22cb1b5
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder.java
@@ -0,0 +1,111 @@
+package org.apache.logging.log4j.core.layout;
+
+import org.apache.logging.log4j.status.StatusLogger;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.util.Objects;
+
+/**
+ * Encoder for StringBuilders that uses ThreadLocals to avoid locking as much as possible.
+ */
+public class StringBuilderEncoder implements Encoder<StringBuilder> {
+
+ private static final int DEFAULT_BYTE_BUFFER_SIZE = 8 * 1024;
+ private final ThreadLocal<CharBuffer> charBufferThreadLocal = new ThreadLocal<>();
+ private final ThreadLocal<ByteBuffer> byteBufferThreadLocal = new ThreadLocal<>();
+ private final ThreadLocal<CharsetEncoder> charsetEncoderThreadLocal = new ThreadLocal<>();
+ private final Charset charset;
+ private final int charBufferSize;
+ private final int byteBufferSize;
+
+ public StringBuilderEncoder(final Charset charset) {
+ this(charset, TextEncoderHelper.DEFAULT_CHAR_BUFFER_SIZE, DEFAULT_BYTE_BUFFER_SIZE);
+ }
+
+ public StringBuilderEncoder(final Charset charset, final int charBufferSize, final int byteBufferSize) {
+ this.charBufferSize = charBufferSize;
+ this.byteBufferSize = byteBufferSize;
+ this.charset = Objects.requireNonNull(charset, "charset");
+ }
+
+ @Override
+ public void encode(final StringBuilder source, final ByteBufferDestination destination) {
+ final ByteBuffer temp = getByteBuffer();
+ temp.clear();
+ temp.limit(Math.min(temp.capacity(), destination.getByteBuffer().capacity()));
+ final CharsetEncoder charsetEncoder = getCharsetEncoder();
+
+ final int estimatedBytes = estimateBytes(source.length(), charsetEncoder.maxBytesPerChar());
+ if (temp.remaining() < estimatedBytes) {
+ encodeSynchronized(getCharsetEncoder(), getCharBuffer(), source, destination);
+ } else {
+ encodeWithThreadLocals(charsetEncoder, getCharBuffer(), temp, source, destination);
+ }
+ }
+
+ private void encodeWithThreadLocals(final CharsetEncoder charsetEncoder, final CharBuffer charBuffer,
+ final ByteBuffer temp, final StringBuilder source, final ByteBufferDestination destination) {
+ try {
+ TextEncoderHelper.encodeTextWithCopy(charsetEncoder, charBuffer, temp, source, destination);
+ } catch (final Exception ex) {
+ ex.printStackTrace();
+ logEncodeTextException(ex, source, destination);
+ TextEncoderHelper.encodeTextFallBack(charset, source, destination);
+ }
+ }
+
+ private static int estimateBytes(final int charCount, final float maxBytesPerChar) {
+ return (int) (charCount * (double) maxBytesPerChar);
+ }
+
+ private void encodeSynchronized(final CharsetEncoder charsetEncoder, final CharBuffer charBuffer,
+ final StringBuilder source, final ByteBufferDestination destination) {
+ synchronized (destination) {
+ try {
+ TextEncoderHelper.encodeText(charsetEncoder, charBuffer, destination.getByteBuffer(), source,
+ destination);
+ } catch (final Exception ex) {
+ logEncodeTextException(ex, source, destination);
+ TextEncoderHelper.encodeTextFallBack(charset, source, destination);
+ }
+ }
+ }
+
+ private CharsetEncoder getCharsetEncoder() {
+ CharsetEncoder result = charsetEncoderThreadLocal.get();
+ if (result == null) {
+ result = charset.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
+ .onUnmappableCharacter(CodingErrorAction.REPLACE);
+ charsetEncoderThreadLocal.set(result);
+ }
+ return result;
+ }
+
+
+ private CharBuffer getCharBuffer() {
+ CharBuffer result = charBufferThreadLocal.get();
+ if (result == null) {
+ result = CharBuffer.wrap(new char[charBufferSize]);
+ charBufferThreadLocal.set(result);
+ }
+ return result;
+ }
+
+ private ByteBuffer getByteBuffer() {
+ ByteBuffer result = byteBufferThreadLocal.get();
+ if (result == null) {
+ result = ByteBuffer.wrap(new byte[byteBufferSize]);
+ byteBufferThreadLocal.set(result);
+ }
+ return result;
+ }
+
+ private void logEncodeTextException(final Exception ex, final StringBuilder text,
+ final ByteBufferDestination destination) {
+ StatusLogger.getLogger().error("Recovering from StringBuilderEncoder.encode('{}') error: {}", text, ex, ex);
+ }
+}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
index 3e59c8a..136e2e6 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
@@ -22,7 +22,6 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
-import java.nio.charset.CodingErrorAction;
import java.util.Objects;
import org.apache.logging.log4j.status.StatusLogger;
@@ -33,43 +32,46 @@ import org.apache.logging.log4j.status.StatusLogger;
* @since 2.6
*/
public class TextEncoderHelper {
- private static final int DEFAULT_BUFFER_SIZE = 2048;
+ static final int DEFAULT_CHAR_BUFFER_SIZE = 2048;
private final Charset charset;
- private final CharBuffer cachedCharBuffer;
- private final CharsetEncoder charsetEncoder;
+// private final CharBuffer cachedCharBuffer;
+// private final CharsetEncoder charsetEncoder;
public TextEncoderHelper(final Charset charset) {
- this(charset, DEFAULT_BUFFER_SIZE);
+ this(charset, DEFAULT_CHAR_BUFFER_SIZE);
}
public TextEncoderHelper(final Charset charset, final int bufferSize) {
this.charset = Objects.requireNonNull(charset, "charset");
- this.charsetEncoder = charset.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- this.cachedCharBuffer = CharBuffer.wrap(new char[bufferSize]);
}
- public void encodeText(final StringBuilder text, final ByteBufferDestination destination) {
+ private void logEncodeTextException(final Exception ex, final StringBuilder text,
+ final ByteBufferDestination destination) {
+ StatusLogger.getLogger().error("Recovering from TextEncoderHelper.encodeText('{}') error", text, ex);
+ }
+
+ static void encodeTextWithCopy(final CharsetEncoder charsetEncoder, final CharBuffer charBuf, final ByteBuffer temp,
+ final StringBuilder text, final ByteBufferDestination destination) {
+ encodeText(charsetEncoder, charBuf, temp, text, destination);
+
synchronized (destination) {
- try {
- encodeText0(text, destination);
- } catch (final Exception ex) {
- logEncodeTextException(ex, text, destination);
- encodeTextFallBack(text, destination);
+ ByteBuffer destinationBuffer = destination.getByteBuffer();
+ if (destinationBuffer != temp) { // still need to write to the destination
+ temp.flip();
+ if (temp.remaining() > destinationBuffer.remaining()) {
+ destinationBuffer = destination.drain(destinationBuffer);
+ }
+ destinationBuffer.put(temp);
+ temp.clear();
}
}
}
- private void logEncodeTextException(final Exception ex, final StringBuilder text,
- final ByteBufferDestination destination) {
- StatusLogger.getLogger().error("Recovering from TextEncoderHelper.encodeText('{}') error", text, ex);
- }
-
- private void encodeText0(final StringBuilder text, final ByteBufferDestination destination) {
+ static void encodeText(final CharsetEncoder charsetEncoder, final CharBuffer charBuf, final ByteBuffer byteBuf,
+ final StringBuilder text, final ByteBufferDestination destination) {
charsetEncoder.reset();
- ByteBuffer byteBuf = destination.getByteBuffer();
- final CharBuffer charBuf = getCachedCharBuffer();
+ ByteBuffer temp = byteBuf; // may be the destination's buffer or a temporary buffer
int start = 0;
int todoChars = text.length();
boolean endOfInput = true;
@@ -81,38 +83,46 @@ public class TextEncoderHelper {
endOfInput = todoChars <= 0;
charBuf.flip(); // prepare for reading: set limit to position, position to zero
- byteBuf = encode(charBuf, endOfInput, destination, byteBuf);
+ temp = encode(charsetEncoder, charBuf, endOfInput, destination, temp);
} while (!endOfInput);
}
- private void encodeTextFallBack(final StringBuilder text, final ByteBufferDestination destination) {
+ static void encodeTextFallBack(final Charset charset, final StringBuilder text,
+ final ByteBufferDestination destination) {
final byte[] bytes = text.toString().getBytes(charset);
- ByteBuffer buffer = destination.getByteBuffer();
- int offset = 0;
- do {
- final int length = Math.min(bytes.length - offset, buffer.remaining());
- buffer.put(bytes, offset, length);
- offset += length;
- if (offset < bytes.length) {
- buffer = destination.drain(buffer);
- }
- } while (offset < bytes.length);
+ synchronized (destination) {
+ ByteBuffer buffer = destination.getByteBuffer();
+ int offset = 0;
+ do {
+ final int length = Math.min(bytes.length - offset, buffer.remaining());
+ buffer.put(bytes, offset, length);
+ offset += length;
+ if (offset < bytes.length) {
+ buffer = destination.drain(buffer);
+ }
+ } while (offset < bytes.length);
+ }
}
- public void encodeText(final CharBuffer charBuf, final ByteBufferDestination destination) {
+ /**
+ * For testing purposes only.
+ */
+ @Deprecated
+ public void encodeText(final CharsetEncoder charsetEncoder, final CharBuffer charBuf,
+ final ByteBufferDestination destination) {
synchronized (destination) {
charsetEncoder.reset();
final ByteBuffer byteBuf = destination.getByteBuffer();
- encode(charBuf, true, destination, byteBuf);
+ encode(charsetEncoder, charBuf, true, destination, byteBuf);
}
}
- private ByteBuffer encode(final CharBuffer charBuf, final boolean endOfInput,
- final ByteBufferDestination destination, ByteBuffer byteBuf) {
+ private static ByteBuffer encode(final CharsetEncoder charsetEncoder, final CharBuffer charBuf,
+ final boolean endOfInput, final ByteBufferDestination destination, ByteBuffer byteBuf) {
try {
- byteBuf = encodeAsMuchAsPossible(charBuf, endOfInput, destination, byteBuf);
+ byteBuf = encodeAsMuchAsPossible(charsetEncoder, charBuf, endOfInput, destination, byteBuf);
if (endOfInput) {
- byteBuf = flushRemainingBytes(destination, byteBuf);
+ byteBuf = flushRemainingBytes(charsetEncoder, destination, byteBuf);
}
} catch (final CharacterCodingException ex) {
throw new IllegalStateException(ex);
@@ -120,39 +130,53 @@ public class TextEncoderHelper {
return byteBuf;
}
- private ByteBuffer encodeAsMuchAsPossible(final CharBuffer charBuf, final boolean endOfInput,
- final ByteBufferDestination destination, ByteBuffer byteBuf) throws CharacterCodingException {
+ private static ByteBuffer encodeAsMuchAsPossible(final CharsetEncoder charsetEncoder, final CharBuffer charBuf,
+ final boolean endOfInput, final ByteBufferDestination destination, ByteBuffer temp)
+ throws CharacterCodingException {
CoderResult result;
do {
- result = charsetEncoder.encode(charBuf, byteBuf, endOfInput);
- if (result.isOverflow()) { // byteBuf full
- // destination consumes contents
- // and returns byte buffer with more capacity
- byteBuf = destination.drain(byteBuf);
- }
- } while (result.isOverflow()); // byteBuf has been drained: retry
+ result = charsetEncoder.encode(charBuf, temp, endOfInput);
+ temp = drainIfByteBufferFull(destination, temp, result);
+ } while (result.isOverflow()); // byte buffer has been drained: retry
if (!result.isUnderflow()) { // we should have fully read the char buffer contents
result.throwException();
}
- return byteBuf;
+ return temp;
+ }
+
+ private static ByteBuffer drainIfByteBufferFull(ByteBufferDestination destination, ByteBuffer temp, CoderResult result) {
+ if (result.isOverflow()) { // byte buffer full
+
+ // SHOULD NOT HAPPEN:
+ // CALLER SHOULD ONLY PASS TEMP ByteBuffer LARGE ENOUGH TO ENCODE ALL CHARACTERS,
+ // AND LOCK ON THE DESTINATION IF THIS IS NOT POSSIBLE
+ ByteBuffer destinationBuffer = destination.getByteBuffer();
+ if (destinationBuffer != temp) {
+ temp.flip();
+ destinationBuffer.put(temp);
+ temp.clear();
+ }
+ // destination consumes contents
+ // and returns byte buffer with more capacity
+ destinationBuffer = destination.drain(destinationBuffer);
+ temp = destinationBuffer;
+ }
+ return temp;
}
- private ByteBuffer flushRemainingBytes(final ByteBufferDestination destination, ByteBuffer byteBuf)
+ private static ByteBuffer flushRemainingBytes(final CharsetEncoder charsetEncoder,
+ final ByteBufferDestination destination, ByteBuffer temp)
throws CharacterCodingException {
CoderResult result;
do {
// write any final bytes to the output buffer once the overall input sequence has been read
- result = charsetEncoder.flush(byteBuf);
- if (result.isOverflow()) { // byteBuf full
- // destination consumes contents
- // and returns byte buffer with more capacity
- byteBuf = destination.drain(byteBuf);
- }
- } while (result.isOverflow()); // byteBuf has been drained: retry
+ result = charsetEncoder.flush(temp);
+ temp = drainIfByteBufferFull(destination, temp, result);
+ } while (result.isOverflow()); // byte buffer has been drained: retry
if (!result.isUnderflow()) { // we should have fully flushed the remaining bytes
result.throwException();
}
- return byteBuf;
+ return temp;
}
/**
@@ -170,8 +194,4 @@ public class TextEncoderHelper {
destination.position(start + length);
return length;
}
-
- CharBuffer getCachedCharBuffer() {
- return cachedCharBuffer;
- }
}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/StringBuilderEncoderTest.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/StringBuilderEncoderTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/StringBuilderEncoderTest.java
new file mode 100644
index 0000000..40c9db8
--- /dev/null
+++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/StringBuilderEncoderTest.java
@@ -0,0 +1,277 @@
+package org.apache.logging.log4j.core.layout;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+
+import org.junit.Test;
+
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests the {@code TextEncoderHelper} class.
+ */
+public class StringBuilderEncoderTest {
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 16, 8 * 1024);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
+
+ for (int i = 0; i < text.length(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 16, 8 * 1024);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(14, 15);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 1, destination.drainPoints.size());
+ assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
+ assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
+ assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
+ assertEquals("destination.buf.pos", text.length() - destination.buffer.capacity(),
+ destination.buffer.position());
+
+ for (int i = 0; i < destination.buffer.capacity(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
+ }
+ for (int i = destination.buffer.capacity(); i < text.length(); i++) {
+ int bufIx = i - destination.buffer.capacity();
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff_MultiplePasses() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 16, 8 * 1024);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(4, 20);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 3, destination.drainPoints.size());
+ assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
+ assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
+ assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
+ assertEquals("drained[1].from", 0, destination.drainPoints.get(1).position);
+ assertEquals("drained[1].to", destination.buffer.capacity(), destination.drainPoints.get(1).limit);
+ assertEquals("drained[1].length", destination.buffer.capacity(), destination.drainPoints.get(1).length());
+ assertEquals("drained[2].from", 0, destination.drainPoints.get(2).position);
+ assertEquals("drained[2].to", destination.buffer.capacity(), destination.drainPoints.get(2).limit);
+ assertEquals("drained[2].length", destination.buffer.capacity(), destination.drainPoints.get(2).length());
+ assertEquals("destination.buf.pos", text.length() - 3 * destination.buffer.capacity(),
+ destination.buffer.position());
+
+ for (int i = 0; i < 3 * destination.buffer.capacity(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
+ }
+ for (int i = 3 * destination.buffer.capacity(); i < text.length(); i++) {
+ int bufIx = i - 3 * destination.buffer.capacity();
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 4, 8 * 1024);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
+
+ for (int i = 0; i < text.length(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 4, 8 * 1024);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i < utf8.length; i++) {
+ assertEquals("byte at " + i, utf8[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
+ final StringBuilderEncoder helper = new StringBuilderEncoder(SHIFT_JIS, 4, 8 * 1024);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] bytes = text.toString().getBytes(SHIFT_JIS);
+ for (int i = 0; i < bytes.length; i++) {
+ assertEquals("byte at " + i, bytes[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 4, 8 * 1024);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 17);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 4, destination.drainPoints.size());
+ assertEquals("destination.buf.pos", 3, destination.buffer.position());
+
+ for (int i = 0; i < text.length() - 3; i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
+ }
+ for (int i = 0; i < 3; i++) {
+ assertEquals("char at " + (12 + i), (byte) text.charAt(12 + i), destination.buffer.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(StandardCharsets.UTF_8, 4, 8 * 1024);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 50);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 7, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i < utf8.length; i++) {
+ assertEquals("byte at " + i, utf8[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
+ final StringBuilderEncoder helper = new StringBuilderEncoder(SHIFT_JIS, 4, 8 * 1024);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 50);
+ helper.encode(text, destination);
+
+ assertEquals("drained", 7, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] bytes = text.toString().getBytes(SHIFT_JIS);
+ for (int i = 0; i < bytes.length; i++) {
+ assertEquals("byte at " + i, bytes[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testCopyCopiesAllDataIfSuffientRemainingSpace() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[16]);
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("everything fits", text.length(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("position moved by length", text.length(), buff.position());
+ }
+
+ @Test
+ public void testCopyUpToRemainingSpace() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[3]);
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("partial copy", buff.capacity(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("no space remaining", 0, buff.remaining());
+ assertEquals("position at end", buff.capacity(), buff.position());
+ }
+
+ @Test
+ public void testCopyDoesNotWriteBeyondStringText() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[5]);
+ assertEquals("initial buffer position", 0, buff.position());
+ final StringBuilder text = createText(2);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("full copy", text.length(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("resulting buffer position", text.length(), buff.position());
+ for (int i = length; i < buff.capacity(); i++) {
+ assertEquals("unset char at " + i, 0, buff.get(i));
+ }
+ }
+
+ @Test
+ public void testCopyStartsAtBufferPosition() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[10]);
+ final int START_POSITION = 5;
+ buff.position(START_POSITION); // set start position
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("partial copy", buff.capacity() - START_POSITION, length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(START_POSITION + i));
+ }
+ assertEquals("buffer position at end", buff.capacity(), buff.position());
+ }
+
+ @Test
+ public void testEncode_ALotWithoutErrors() throws Exception {
+ final StringBuilderEncoder helper = new StringBuilderEncoder(Charset.defaultCharset());
+ final StringBuilder text = new StringBuilder("2016-04-13 21:07:47,487 DEBUG [org.apache.logging.log4j.perf.jmh.FileAppenderBenchmark.log4j2ParameterizedString-jmh-worker-1] FileAppenderBenchmark - This is a debug [2383178] message\r\n");
+ final int DESTINATION_SIZE = 1024 * 1024;
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(256 * 1024, DESTINATION_SIZE);
+
+ int max = DESTINATION_SIZE / text.length();
+ for (int i = 0; i < max; i++) {
+ helper.encode(text, destination);
+ }
+ // no error
+ }
+
+ private StringBuilder createText(final int length) {
+ final StringBuilder result = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ result.append((char) (' ' + i)); // space=0x20
+ }
+ return result;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
deleted file mode 100644
index e98dfca..0000000
--- a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
+++ /dev/null
@@ -1,263 +0,0 @@
-package org.apache.logging.log4j.core.layout;/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache license, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the license for the specific language governing permissions and
- * limitations under the license.
- */
-
-import java.nio.CharBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Tests the {@code TextEncoderHelper} class.
- */
-public class TextEncoderHelperTest {
-
- @Test
- public void testEncodeText_TextFitCharBuff_BytesFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
- final StringBuilder text = createText(15);
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 0, destination.drainPoints.size());
- assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
-
- for (int i = 0; i < text.length(); i++) {
- assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
- }
- }
-
- @Test
- public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
- final StringBuilder text = createText(15);
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(14, 15);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 1, destination.drainPoints.size());
- assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
- assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
- assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
- assertEquals("destination.buf.pos", text.length() - destination.buffer.capacity(),
- destination.buffer.position());
-
- for (int i = 0; i < destination.buffer.capacity(); i++) {
- assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
- }
- for (int i = destination.buffer.capacity(); i < text.length(); i++) {
- int bufIx = i - destination.buffer.capacity();
- assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
- }
- }
-
- @Test
- public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff_MultiplePasses() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
- final StringBuilder text = createText(15);
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(4, 20);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 3, destination.drainPoints.size());
- assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
- assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
- assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
- assertEquals("drained[1].from", 0, destination.drainPoints.get(1).position);
- assertEquals("drained[1].to", destination.buffer.capacity(), destination.drainPoints.get(1).limit);
- assertEquals("drained[1].length", destination.buffer.capacity(), destination.drainPoints.get(1).length());
- assertEquals("drained[2].from", 0, destination.drainPoints.get(2).position);
- assertEquals("drained[2].to", destination.buffer.capacity(), destination.drainPoints.get(2).limit);
- assertEquals("drained[2].length", destination.buffer.capacity(), destination.drainPoints.get(2).length());
- assertEquals("destination.buf.pos", text.length() - 3 * destination.buffer.capacity(),
- destination.buffer.position());
-
- for (int i = 0; i < 3 * destination.buffer.capacity(); i++) {
- assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
- }
- for (int i = 3 * destination.buffer.capacity(); i < text.length(); i++) {
- int bufIx = i - 3 * destination.buffer.capacity();
- assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
- }
- }
-
- @Test
- public void testEncodeText_TextDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
- final StringBuilder text = createText(15);
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 0, destination.drainPoints.size());
- assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
-
- for (int i = 0; i < text.length(); i++) {
- assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
- }
- }
-
- @Test
- public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
- final StringBuilder text = new StringBuilder( // 日本語テスト文章
- "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 0, destination.drainPoints.size());
- destination.drain(destination.getByteBuffer());
-
- final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8);
- for (int i = 0; i < utf8.length; i++) {
- assertEquals("byte at " + i, utf8[i], destination.drained.get(i));
- }
- }
-
- @Test
- public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
- final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
- final TextEncoderHelper helper = new TextEncoderHelper(SHIFT_JIS, 4);
- final StringBuilder text = new StringBuilder( // 日本語テスト文章
- "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 0, destination.drainPoints.size());
- destination.drain(destination.getByteBuffer());
-
- final byte[] bytes = text.toString().getBytes(SHIFT_JIS);
- for (int i = 0; i < bytes.length; i++) {
- assertEquals("byte at " + i, bytes[i], destination.drained.get(i));
- }
- }
-
- @Test
- public void testEncodeText_TextDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
- final StringBuilder text = createText(15);
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 17);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 4, destination.drainPoints.size());
- assertEquals("destination.buf.pos", 3, destination.buffer.position());
-
- for (int i = 0; i < text.length() - 3; i++) {
- assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
- }
- for (int i = 0; i < 3; i++) {
- assertEquals("char at " + (12 + i), (byte) text.charAt(12 + i), destination.buffer.get(i));
- }
- }
-
- @Test
- public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
- final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
- final StringBuilder text = new StringBuilder( // 日本語テスト文章
- "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 50);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 7, destination.drainPoints.size());
- destination.drain(destination.getByteBuffer());
-
- final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8);
- for (int i = 0; i < utf8.length; i++) {
- assertEquals("byte at " + i, utf8[i], destination.drained.get(i));
- }
- }
-
- @Test
- public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
- final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
- final TextEncoderHelper helper = new TextEncoderHelper(SHIFT_JIS, 4);
- final StringBuilder text = new StringBuilder( // 日本語テスト文章
- "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
- final SpyByteBufferDestination destination = new SpyByteBufferDestination(3, 50);
- helper.encodeText(text, destination);
-
- assertEquals("drained", 7, destination.drainPoints.size());
- destination.drain(destination.getByteBuffer());
-
- final byte[] bytes = text.toString().getBytes(SHIFT_JIS);
- for (int i = 0; i < bytes.length; i++) {
- assertEquals("byte at " + i, bytes[i], destination.drained.get(i));
- }
- }
-
- @Test
- public void testCopyCopiesAllDataIfSuffientRemainingSpace() throws Exception {
- final CharBuffer buff = CharBuffer.wrap(new char[16]);
- final StringBuilder text = createText(15);
- final int length = TextEncoderHelper.copy(text, 0, buff);
- assertEquals("everything fits", text.length(), length);
- for (int i = 0; i < length; i++) {
- assertEquals("char at " + i, text.charAt(i), buff.get(i));
- }
- assertEquals("position moved by length", text.length(), buff.position());
- }
-
- @Test
- public void testCopyUpToRemainingSpace() throws Exception {
- final CharBuffer buff = CharBuffer.wrap(new char[3]);
- final StringBuilder text = createText(15);
- final int length = TextEncoderHelper.copy(text, 0, buff);
- assertEquals("partial copy", buff.capacity(), length);
- for (int i = 0; i < length; i++) {
- assertEquals("char at " + i, text.charAt(i), buff.get(i));
- }
- assertEquals("no space remaining", 0, buff.remaining());
- assertEquals("position at end", buff.capacity(), buff.position());
- }
-
- @Test
- public void testCopyDoesNotWriteBeyondStringText() throws Exception {
- final CharBuffer buff = CharBuffer.wrap(new char[5]);
- assertEquals("initial buffer position", 0, buff.position());
- final StringBuilder text = createText(2);
- final int length = TextEncoderHelper.copy(text, 0, buff);
- assertEquals("full copy", text.length(), length);
- for (int i = 0; i < length; i++) {
- assertEquals("char at " + i, text.charAt(i), buff.get(i));
- }
- assertEquals("resulting buffer position", text.length(), buff.position());
- for (int i = length; i < buff.capacity(); i++) {
- assertEquals("unset char at " + i, 0, buff.get(i));
- }
- }
-
- @Test
- public void testCopyStartsAtBufferPosition() throws Exception {
- final CharBuffer buff = CharBuffer.wrap(new char[10]);
- final int START_POSITION = 5;
- buff.position(START_POSITION); // set start position
- final StringBuilder text = createText(15);
- final int length = TextEncoderHelper.copy(text, 0, buff);
- assertEquals("partial copy", buff.capacity() - START_POSITION, length);
- for (int i = 0; i < length; i++) {
- assertEquals("char at " + i, text.charAt(i), buff.get(START_POSITION + i));
- }
- assertEquals("buffer position at end", buff.capacity(), buff.position());
- }
-
- private StringBuilder createText(final int length) {
- final StringBuilder result = new StringBuilder(length);
- for (int i = 0; i < length; i++) {
- result.append((char) (' ' + i)); // space=0x20
- }
- return result;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/TextEncoderHelperBenchmark.java
----------------------------------------------------------------------
diff --git a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/TextEncoderHelperBenchmark.java b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/TextEncoderHelperBenchmark.java
index bab01ff..888035d 100644
--- a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/TextEncoderHelperBenchmark.java
+++ b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/TextEncoderHelperBenchmark.java
@@ -29,7 +29,7 @@ import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.impl.Log4jLogEvent;
import org.apache.logging.log4j.core.layout.ByteBufferDestination;
import org.apache.logging.log4j.core.layout.PatternLayout;
-import org.apache.logging.log4j.core.layout.TextEncoderHelper;
+import org.apache.logging.log4j.core.layout.StringBuilderEncoder;
import org.apache.logging.log4j.message.Message;
import org.apache.logging.log4j.message.SimpleMessage;
import org.openjdk.jmh.annotations.Benchmark;
@@ -40,13 +40,13 @@ import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
/**
- * Tests Log4j2 TextEncoderHelper performance.
+ * Tests Log4j2 StringBuilderEncoder performance.
*/
// ============================== HOW TO RUN THIS TEST: ====================================
//
// single thread:
-// java -Dfile.encoding=ISO-8859-1 -Dlog4j2.is.webapp=false -Dlog4j2.enable.threadlocals=true -jar log4j-perf/target/benchmarks.jar ".*TextEncoderHelper.*" -f 1 -wi 5 -i 10
-// java -Dfile.encoding=UTF8 -Dlog4j2.is.webapp=false -Dlog4j2.enable.threadlocals=true -jar log4j-perf/target/benchmarks.jar ".*TextEncoderHelper.*" -f 1 -wi 5 -i 10
+// java -Dfile.encoding=ISO-8859-1 -Dlog4j2.is.webapp=false -Dlog4j2.enable.threadlocals=true -jar log4j-perf/target/benchmarks.jar ".*StringBuilderEncoder.*" -f 1 -wi 5 -i 10
+// java -Dfile.encoding=UTF8 -Dlog4j2.is.webapp=false -Dlog4j2.enable.threadlocals=true -jar log4j-perf/target/benchmarks.jar ".*StringBuilderEncoder.*" -f 1 -wi 5 -i 10
//
// Usage help:
// java -jar log4j-perf/target/benchmarks.jar -help
@@ -116,13 +116,10 @@ public class TextEncoderHelperBenchmark {
return STR_TEXT.getBytes();
}
- private static final ThreadLocal<TextEncoderHelper> textEncoderHelper = new ThreadLocal<>();
- private TextEncoderHelper getCachedTextEncoderHelper() {
- TextEncoderHelper result = textEncoderHelper.get();
- if (result == null) {
- result = new TextEncoderHelper(CHARSET_DEFAULT);
- textEncoderHelper.set(result);
- }
+ //private static final ThreadLocal<StringBuilderEncoder> textEncoderHelper = new ThreadLocal<>();
+ private StringBuilderEncoder textEncoderHelper = new StringBuilderEncoder(CHARSET_DEFAULT);
+ private StringBuilderEncoder getEncoder() {
+ StringBuilderEncoder result = textEncoderHelper;
return result;
}
@@ -130,49 +127,49 @@ public class TextEncoderHelperBenchmark {
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public long textEncoderEncode() {
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- helper.encodeText(BUFF_TEXT, destination);
-
- return destination.count;
- }
-
- @Benchmark
- @BenchmarkMode(Mode.SampleTime)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
- public long charBufferEncode() {
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- CHAR_BUFFER.limit(CHAR_BUFFER.capacity());
- CHAR_BUFFER.position(0);
- helper.encodeText(CHAR_BUFFER, destination);
+ final StringBuilderEncoder helper = getEncoder();
+ helper.encode(BUFF_TEXT, destination);
return destination.count;
}
- @Benchmark
- @BenchmarkMode(Mode.SampleTime)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
- public long charBufferCopyAndEncode() {
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- CHAR_BUFFER.clear();
- CHAR_BUFFER.put(STR);
- CHAR_BUFFER.flip();
- helper.encodeText(CHAR_BUFFER, destination);
-
- return destination.count;
- }
-
- @Benchmark
- @BenchmarkMode(Mode.SampleTime)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
- public long textHelperCopyAndEncode() {
- final TextEncoderHelper helper = getCachedTextEncoderHelper();
- CHAR_BUFFER.clear();
- copy(BUFF_TEXT, 0, CHAR_BUFFER);
- CHAR_BUFFER.flip();
- helper.encodeText(CHAR_BUFFER, destination);
-
- return destination.count;
- }
+// @Benchmark
+// @BenchmarkMode(Mode.SampleTime)
+// @OutputTimeUnit(TimeUnit.NANOSECONDS)
+// public long charBufferEncode() {
+// final StringBuilderEncoder helper = getEncoder();
+// CHAR_BUFFER.limit(CHAR_BUFFER.capacity());
+// CHAR_BUFFER.position(0);
+// helper.encode(CHAR_BUFFER, destination);
+//
+// return destination.count;
+// }
+//
+// @Benchmark
+// @BenchmarkMode(Mode.SampleTime)
+// @OutputTimeUnit(TimeUnit.NANOSECONDS)
+// public long charBufferCopyAndEncode() {
+// final StringBuilderEncoder helper = getEncoder();
+// CHAR_BUFFER.clear();
+// CHAR_BUFFER.put(STR);
+// CHAR_BUFFER.flip();
+// helper.encode(CHAR_BUFFER, destination);
+//
+// return destination.count;
+// }
+//
+// @Benchmark
+// @BenchmarkMode(Mode.SampleTime)
+// @OutputTimeUnit(TimeUnit.NANOSECONDS)
+// public long textHelperCopyAndEncode() {
+// final StringBuilderEncoder helper = getEncoder();
+// CHAR_BUFFER.clear();
+// copy(BUFF_TEXT, 0, CHAR_BUFFER);
+// CHAR_BUFFER.flip();
+// helper.encode(CHAR_BUFFER, destination);
+//
+// return destination.count;
+// }
/**
* Copies characters from the CharSequence into the CharBuffer,
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/23cd33fc/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/NoGcLayout.java
----------------------------------------------------------------------
diff --git a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/NoGcLayout.java b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/NoGcLayout.java
index 345a231..4b1a4ad 100644
--- a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/NoGcLayout.java
+++ b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/NoGcLayout.java
@@ -20,6 +20,7 @@ import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.layout.ByteBufferDestination;
import org.apache.logging.log4j.core.layout.Encoder;
+import org.apache.logging.log4j.core.layout.StringBuilderEncoder;
import org.apache.logging.log4j.core.layout.TextEncoderHelper;
import org.apache.logging.log4j.core.pattern.FormattingInfo;
import org.apache.logging.log4j.core.pattern.PatternFormatter;
@@ -39,18 +40,18 @@ import java.util.Map;
public class NoGcLayout implements Layout<Serializable>, Encoder<LogEvent> {
private final StringBuilder cachedStringBuilder = new StringBuilder(2048);
private final PatternSerializer2 serializer = new PatternSerializer2();
- private final TextEncoderHelper cachedHelper;
+ private final StringBuilderEncoder cachedHelper;
public NoGcLayout(Charset charset) {
- cachedHelper = new TextEncoderHelper(charset);
+ cachedHelper = new StringBuilderEncoder(charset);
}
@Override
public void encode(LogEvent event, ByteBufferDestination destination) {
StringBuilder text = toText(event, getCachedStringBuilder());
- TextEncoderHelper helper = getCachedHelper();
- helper.encodeText(text, destination);
+ Encoder<StringBuilder> helper = getCachedHelper();
+ helper.encode(text, destination);
}
/**
@@ -69,7 +70,7 @@ public class NoGcLayout implements Layout<Serializable>, Encoder<LogEvent> {
return cachedStringBuilder;
}
- public TextEncoderHelper getCachedHelper() {
+ public Encoder<StringBuilder> getCachedHelper() {
return cachedHelper;
}
[2/2] logging-log4j2 git commit: LOG4J2-1274 TextEncoderHelper cleanup
Posted by rp...@apache.org.
LOG4J2-1274 TextEncoderHelper cleanup
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/0a06f8f0
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/0a06f8f0
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/0a06f8f0
Branch: refs/heads/master
Commit: 0a06f8f0dea40cce08d987da3cc6e340835060d2
Parents: 23cd33f
Author: rpopma <rp...@apache.org>
Authored: Thu Apr 14 01:23:18 2016 +0900
Committer: rpopma <rp...@apache.org>
Committed: Thu Apr 14 01:23:18 2016 +0900
----------------------------------------------------------------------
.../log4j/core/layout/TextEncoderHelper.java | 55 ++++++++------------
1 file changed, 21 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0a06f8f0/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
index 136e2e6..09a49d8 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
@@ -22,9 +22,6 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
-import java.util.Objects;
-
-import org.apache.logging.log4j.status.StatusLogger;
/**
* Helper class to encode text to binary data without allocating temporary objects.
@@ -34,27 +31,34 @@ import org.apache.logging.log4j.status.StatusLogger;
public class TextEncoderHelper {
static final int DEFAULT_CHAR_BUFFER_SIZE = 2048;
- private final Charset charset;
-// private final CharBuffer cachedCharBuffer;
-// private final CharsetEncoder charsetEncoder;
-
- public TextEncoderHelper(final Charset charset) {
- this(charset, DEFAULT_CHAR_BUFFER_SIZE);
+ private TextEncoderHelper() {
}
- public TextEncoderHelper(final Charset charset, final int bufferSize) {
- this.charset = Objects.requireNonNull(charset, "charset");
- }
-
- private void logEncodeTextException(final Exception ex, final StringBuilder text,
- final ByteBufferDestination destination) {
- StatusLogger.getLogger().error("Recovering from TextEncoderHelper.encodeText('{}') error", text, ex);
+ static void encodeTextFallBack(final Charset charset, final StringBuilder text,
+ final ByteBufferDestination destination) {
+ final byte[] bytes = text.toString().getBytes(charset);
+ synchronized (destination) {
+ ByteBuffer buffer = destination.getByteBuffer();
+ int offset = 0;
+ do {
+ final int length = Math.min(bytes.length - offset, buffer.remaining());
+ buffer.put(bytes, offset, length);
+ offset += length;
+ if (offset < bytes.length) {
+ buffer = destination.drain(buffer);
+ }
+ } while (offset < bytes.length);
+ }
}
static void encodeTextWithCopy(final CharsetEncoder charsetEncoder, final CharBuffer charBuf, final ByteBuffer temp,
final StringBuilder text, final ByteBufferDestination destination) {
+
encodeText(charsetEncoder, charBuf, temp, text, destination);
+ copyDataToDestination(temp, destination);
+ }
+ private static void copyDataToDestination(final ByteBuffer temp, final ByteBufferDestination destination) {
synchronized (destination) {
ByteBuffer destinationBuffer = destination.getByteBuffer();
if (destinationBuffer != temp) { // still need to write to the destination
@@ -87,28 +91,11 @@ public class TextEncoderHelper {
} while (!endOfInput);
}
- static void encodeTextFallBack(final Charset charset, final StringBuilder text,
- final ByteBufferDestination destination) {
- final byte[] bytes = text.toString().getBytes(charset);
- synchronized (destination) {
- ByteBuffer buffer = destination.getByteBuffer();
- int offset = 0;
- do {
- final int length = Math.min(bytes.length - offset, buffer.remaining());
- buffer.put(bytes, offset, length);
- offset += length;
- if (offset < bytes.length) {
- buffer = destination.drain(buffer);
- }
- } while (offset < bytes.length);
- }
- }
-
/**
* For testing purposes only.
*/
@Deprecated
- public void encodeText(final CharsetEncoder charsetEncoder, final CharBuffer charBuf,
+ public static void encodeText(final CharsetEncoder charsetEncoder, final CharBuffer charBuf,
final ByteBufferDestination destination) {
synchronized (destination) {
charsetEncoder.reset();