You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rp...@apache.org on 2016/02/24 00:20:50 UTC
[02/50] logging-log4j2 git commit: LOG4J2-1274 add TextEncoderHelper
with initial unit tests
LOG4J2-1274 add TextEncoderHelper with initial unit tests
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/27619e08
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/27619e08
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/27619e08
Branch: refs/heads/LOG4J2-1278-gc-free-logger
Commit: 27619e0812ceaf4ead6b086268505ee55459e7eb
Parents: 9594f5b
Author: rpopma <rp...@apache.org>
Authored: Sun Feb 21 16:49:31 2016 +0900
Committer: rpopma <rp...@apache.org>
Committed: Sun Feb 21 16:49:31 2016 +0900
----------------------------------------------------------------------
.../log4j/core/layout/TextEncoderHelper.java | 136 ++++++++++++
.../core/layout/SpyByteBufferDestination.java | 63 ++++++
.../core/layout/TextEncoderHelperTest.java | 216 +++++++++++++++++++
.../apache/logging/log4j/perf/nogc/Encoder.java | 24 ---
4 files changed, 415 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
new file mode 100644
index 0000000..79f0123
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.util.Objects;
+
+/**
+ * Helper class to encode text to binary data without allocating temporary objects.
+ *
+ * @since 2.6
+ */
+class TextEncoderHelper {
+ private static final int DEFAULT_BUFFER_SIZE = 2048;
+
+ private final Charset charset;
+ private final CharBuffer cachedCharBuffer;
+ private final CharsetEncoder charsetEncoder;
+
+ public TextEncoderHelper(final Charset charset) {
+ this(charset, DEFAULT_BUFFER_SIZE);
+ }
+
+ public TextEncoderHelper(final Charset charset, final int bufferSize) {
+ this.charset = Objects.requireNonNull(charset, "charset");
+ this.charsetEncoder = charset.newEncoder().onMalformedInput(CodingErrorAction.REPLACE)
+ .onUnmappableCharacter(CodingErrorAction.REPLACE);
+ this.cachedCharBuffer = CharBuffer.wrap(new char[bufferSize]);
+ }
+
+ public void encodeText(final StringBuilder text, final ByteBufferDestination destination) {
+ charsetEncoder.reset();
+ ByteBuffer byteBuf = destination.getByteBuffer();
+ final CharBuffer charBuf = getCachedCharBuffer();
+ int start = 0;
+ int todoChars = text.length();
+ boolean endOfInput = true;
+ do {
+ charBuf.clear(); // reset character buffer position to zero, limit to capacity
+ int copied = copy(text, start, charBuf);
+ start += copied;
+ todoChars -= copied;
+ endOfInput = todoChars <= 0;
+
+ charBuf.flip(); // prepare for reading: set limit to position, position to zero
+ byteBuf = encode(charBuf, endOfInput, destination, byteBuf);
+ } while (!endOfInput);
+ }
+
+ private ByteBuffer encode(final CharBuffer charBuf, final boolean endOfInput,
+ final ByteBufferDestination destination, ByteBuffer byteBuf) {
+ try {
+ byteBuf = encodeAsMuchAsPossible(charBuf, endOfInput, destination, byteBuf);
+ if (endOfInput) {
+ byteBuf = flushRemainingBytes(destination, byteBuf);
+ }
+ } catch (final CharacterCodingException ex) {
+ throw new IllegalStateException(ex);
+ }
+ return byteBuf;
+ }
+
+ private ByteBuffer encodeAsMuchAsPossible(final CharBuffer charBuf, final boolean endOfInput,
+ final ByteBufferDestination destination, ByteBuffer byteBuf) throws CharacterCodingException {
+ CoderResult result;
+ do {
+ result = charsetEncoder.encode(charBuf, byteBuf, endOfInput);
+ if (result.isOverflow()) { // byteBuf full
+ // destination consumes contents
+ // and returns byte buffer with more capacity
+ byteBuf = destination.drain(byteBuf);
+ }
+ } while (result.isOverflow()); // byteBuf has been drained: retry
+ if (!result.isUnderflow()) { // we should have fully read the char buffer contents
+ result.throwException();
+ }
+ return byteBuf;
+ }
+
+ private ByteBuffer flushRemainingBytes(final ByteBufferDestination destination, ByteBuffer byteBuf)
+ throws CharacterCodingException {
+ CoderResult result;
+ do {
+ // write any final bytes to the output buffer once the overall input sequence has been read
+ result = charsetEncoder.flush(byteBuf);
+ if (result.isOverflow()) { // byteBuf full
+ // destination consumes contents
+ // and returns byte buffer with more capacity
+ byteBuf = destination.drain(byteBuf);
+ }
+ } while (result.isOverflow()); // byteBuf has been drained: retry
+ if (!result.isUnderflow()) { // we should have fully flushed the remaining bytes
+ result.throwException();
+ }
+ return byteBuf;
+ }
+
+ /**
+ * Copies characters from the StringBuilder into the CharBuffer,
+ * starting at the specified offset and ending when either all
+ * characters have been copied or when the CharBuffer is full.
+ *
+ * @return the number of characters that were copied
+ */
+ static int copy(final StringBuilder source, final int offset, final CharBuffer destination) {
+ final int length = Math.min(source.length() - offset, destination.remaining());
+ for (int i = offset; i < offset + length; i++) {
+ destination.put(source.charAt(i));
+ }
+ return length;
+ }
+
+ CharBuffer getCachedCharBuffer() {
+ return cachedCharBuffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java
new file mode 100644
index 0000000..5d8f3b5
--- /dev/null
+++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ByteBufferDestination for test purposes that counts how much data was added.
+ */
+public class SpyByteBufferDestination implements ByteBufferDestination {
+ public final ByteBuffer buffer;
+ public final ByteBuffer drained;
+ public final List<Data> drainPoints = new ArrayList<>();
+
+ public static class Data {
+ public final int position;
+ public final int limit;
+
+ public Data(final int position, final int limit) {
+ this.position = position;
+ this.limit = limit;
+ }
+
+ public int length() {
+ return limit - position;
+ }
+ }
+
+ public SpyByteBufferDestination(int bufferSize, int destinationSize) {
+ buffer = ByteBuffer.wrap(new byte[bufferSize]);
+ drained = ByteBuffer.wrap(new byte[destinationSize]);
+ }
+
+ @Override
+ public ByteBuffer getByteBuffer() {
+ return buffer;
+ }
+
+ @Override
+ public ByteBuffer drain(final ByteBuffer buf) {
+ buf.flip();
+ drainPoints.add(new Data(buf.position(), buf.limit()));
+ drained.put(buf);
+ buf.clear();
+ return buf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
new file mode 100644
index 0000000..5ba6a33
--- /dev/null
+++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java
@@ -0,0 +1,216 @@
+package org.apache.logging.log4j.core.layout;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests the {@code TextEncoderHelper} class.
+ */
+public class TextEncoderHelperTest {
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesFitByteBuff() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
+
+ for (int i = 0; i < text.length(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(14, 15);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 1, destination.drainPoints.size());
+ assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
+ assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
+ assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
+ assertEquals("destination.buf.pos", text.length() - destination.buffer.capacity(),
+ destination.buffer.position());
+
+ for (int i = 0; i < destination.buffer.capacity(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
+ }
+ for (int i = destination.buffer.capacity(); i < text.length(); i++) {
+ int bufIx = i - destination.buffer.capacity();
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff_MultiplePasses() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(4, 20);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 3, destination.drainPoints.size());
+ assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position);
+ assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit);
+ assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length());
+ assertEquals("drained[1].from", 0, destination.drainPoints.get(1).position);
+ assertEquals("drained[1].to", destination.buffer.capacity(), destination.drainPoints.get(1).limit);
+ assertEquals("drained[1].length", destination.buffer.capacity(), destination.drainPoints.get(1).length());
+ assertEquals("drained[2].from", 0, destination.drainPoints.get(2).position);
+ assertEquals("drained[2].to", destination.buffer.capacity(), destination.drainPoints.get(2).limit);
+ assertEquals("drained[2].length", destination.buffer.capacity(), destination.drainPoints.get(2).length());
+ assertEquals("destination.buf.pos", text.length() - 3 * destination.buffer.capacity(),
+ destination.buffer.position());
+
+ for (int i = 0; i < 3 * destination.buffer.capacity(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i));
+ }
+ for (int i = 3 * destination.buffer.capacity(); i < text.length(); i++) {
+ int bufIx = i - 3 * destination.buffer.capacity();
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
+ final StringBuilder text = createText(15);
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ assertEquals("destination.buf.pos", text.length(), destination.buffer.position());
+
+ for (int i = 0; i < text.length(); i++) {
+ assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i < utf8.length; i++) {
+ assertEquals("byte at " + i, utf8[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesFitByteBuff() throws Exception {
+ final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
+ final TextEncoderHelper helper = new TextEncoderHelper(SHIFT_JIS, 4);
+ final StringBuilder text = new StringBuilder( // 日本語テスト文章
+ "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0");
+ final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50);
+ helper.encodeText(text, destination);
+
+ assertEquals("drained", 0, destination.drainPoints.size());
+ destination.drain(destination.getByteBuffer());
+
+ final byte[] bytes = text.toString().getBytes(SHIFT_JIS);
+ for (int i = 0; i < bytes.length; i++) {
+ assertEquals("byte at " + i, bytes[i], destination.drained.get(i));
+ }
+ }
+
+ @Test
+ public void testEncodeText_TextDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception {
+ final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16);
+ // TODO
+ }
+
+ @Test
+ public void testCopyCopiesAllDataIfSuffientRemainingSpace() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[16]);
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("everything fits", text.length(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("position moved by length", text.length(), buff.position());
+ }
+
+ @Test
+ public void testCopyUpToRemainingSpace() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[3]);
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("partial copy", buff.capacity(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("no space remaining", 0, buff.remaining());
+ assertEquals("position at end", buff.capacity(), buff.position());
+ }
+
+ @Test
+ public void testCopyDoesNotWriteBeyondStringText() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[5]);
+ assertEquals("initial buffer position", 0, buff.position());
+ final StringBuilder text = createText(2);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("full copy", text.length(), length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(i));
+ }
+ assertEquals("resulting buffer position", text.length(), buff.position());
+ for (int i = length; i < buff.capacity(); i++) {
+ assertEquals("unset char at " + i, 0, buff.get(i));
+ }
+ }
+
+ @Test
+ public void testCopyStartsAtBufferPosition() throws Exception {
+ final CharBuffer buff = CharBuffer.wrap(new char[10]);
+ final int START_POSITION = 5;
+ buff.position(START_POSITION); // set start position
+ final StringBuilder text = createText(15);
+ final int length = TextEncoderHelper.copy(text, 0, buff);
+ assertEquals("partial copy", buff.capacity() - START_POSITION, length);
+ for (int i = 0; i < length; i++) {
+ assertEquals("char at " + i, text.charAt(i), buff.get(START_POSITION + i));
+ }
+ assertEquals("buffer position at end", buff.capacity(), buff.position());
+ }
+
+ private StringBuilder createText(final int length) {
+ final StringBuilder result = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ result.append((char) (' ' + i)); // space=0x20
+ }
+ return result;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java
----------------------------------------------------------------------
diff --git a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java
deleted file mode 100644
index 9a4da17..0000000
--- a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache license, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the license for the specific language governing permissions and
- * limitations under the license.
- */
-package org.apache.logging.log4j.perf.nogc;
-
-/**
- * Encoder interface proposed in LOG4J2-1274..
- */
-public interface Encoder<T> {
- void encode(T source, ByteBufferDestination destination);
-}