You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rp...@apache.org on 2014/12/27 14:29:42 UTC
[1/2] logging-log4j2 git commit: various small improvements
Repository: logging-log4j2
Updated Branches:
refs/heads/master 914f5e1a2 -> 50278c35b
various small improvements
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/ae89596f
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/ae89596f
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/ae89596f
Branch: refs/heads/master
Commit: ae89596f04ffd82b0ef9c2645eda8b0ccae24d68
Parents: 914f5e1
Author: rpopma <rp...@apache.org>
Authored: Sat Dec 27 22:10:35 2014 +0900
Committer: rpopma <rp...@apache.org>
Committed: Sat Dec 27 22:10:35 2014 +0900
----------------------------------------------------------------------
.../log4j/core/async/perftest/PerfTest.java | 85 ++++++++++++--------
1 file changed, 53 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/ae89596f/log4j-core/src/test/java/org/apache/logging/log4j/core/async/perftest/PerfTest.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/async/perftest/PerfTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/async/perftest/PerfTest.java
index 60d88f5..c792db5 100644
--- a/log4j-core/src/test/java/org/apache/logging/log4j/core/async/perftest/PerfTest.java
+++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/async/perftest/PerfTest.java
@@ -18,16 +18,24 @@ package org.apache.logging.log4j.core.async.perftest;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
import org.apache.logging.log4j.core.util.Loader;
import com.lmax.disruptor.collections.Histogram;
+/**
+ * Single-threaded performance test. Usually invoked from PerfTestDriver as part of a series of tests.
+ * <p>
+ * To run a single instance of this class for the log4j2 test runner:<br>
+ * java -Dlog4j.configurationFile=mylog4j2.xml org.apache.logging.log4j.core.async.perftest.PerfTest \
+ * org.apache.logging.log4j.core.async.perftest.RunLog4j2 <name> <resultfile.txt> <-verbose> <-throughput>
+ */
public class PerfTest {
private static final String LINE100 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890!\"#$%&'()-=^~|\\@`[]{};:+*,.<>/?_123456";
- public static final String LINE500 = LINE100 + LINE100 + LINE100 + LINE100
- + LINE100;
+ public static final String LINE500 = LINE100 + LINE100 + LINE100 + LINE100 + LINE100;
static boolean verbose = false;
static boolean throughput;
@@ -73,7 +81,7 @@ public class PerfTest {
final String resultFile = args.length > 2 ? args[2] : null;
for (final String arg : args) {
if (verbose && throughput) {
- break;
+ break;
}
if ("-verbose".equalsIgnoreCase(arg)) {
verbose = true;
@@ -82,60 +90,73 @@ public class PerfTest {
throughput = true;
}
}
- final int threadCount = args.length > 2 ? Integer.parseInt(args[3]) : 3;
- printf("Starting %s %s (%d)...%n", getClass().getSimpleName(), name,
- threadCount);
+ final int threadCount = 1;
+ printf("Starting %s %s (%d)...%n", getClass().getSimpleName(), name, threadCount);
runTestAndPrintResult(runner, name, threadCount, resultFile);
runner.shutdown();
System.exit(0);
}
- public void runTestAndPrintResult(final IPerfTestRunner runner,
- final String name, final int threadCount, final String resultFile)
- throws Exception {
+ public void runTestAndPrintResult(final IPerfTestRunner runner, final String name, final int threadCount,
+ final String resultFile) throws Exception {
final Histogram warmupHist = createHistogram();
// ThreadContext.put("aKey", "mdcVal");
println("Warming up the JVM...");
final long t1 = System.nanoTime();
- // warmup at least 2 rounds and at most 1 minute
- final long stop = System.currentTimeMillis() + (60 * 1000);
- for (int i = 0; i < 10; i++) {
- final int LINES = throughput ? 50000 : 200000;
+ // warmup at least 10 seconds
+ final int LINES = 50000;
+ int iterations = 0;
+ final long stop = System.currentTimeMillis() + (10 * 1000); // 10 seconds
+ do {
runTest(runner, LINES, null, warmupHist, 1);
- if (i > 0 && System.currentTimeMillis() >= stop) {
- return;
- }
- }
+ iterations++;
+ } while (System.currentTimeMillis() < stop);
- printf("Warmup complete in %.1f seconds%n", (System.nanoTime() - t1)
- / (1000.0 * 1000.0 * 1000.0));
+ printf("Warmup complete in %.1f seconds (%d iterations)%n", (System.nanoTime() - t1)
+ / (1000.0 * 1000.0 * 1000.0), iterations);
println("Waiting 10 seconds for buffers to drain warmup data...");
- Thread.sleep(10000);
- println("Starting the main test...");
- // test
- throughput = false;
- runSingleThreadedTest(runner, name, resultFile);
+ Thread.sleep(3000);
+ //forceRemap(LINES, iterations, runner);
+ Thread.sleep(7000);
+ println("Starting the main test...");
+ runSingleThreadedTest(runner, LINES * iterations, name, resultFile);
Thread.sleep(1000);
+ }
- throughput = true;
- runSingleThreadedTest(runner, name, resultFile);
+ /**
+ * Log some extra bytes to fill the memory mapped buffer to force it to remap.
+ */
+ private void forceRemap(final int linesPerIteration, final int iterations, final IPerfTestRunner runner) {
+ final int LINESEP = System.getProperty("line.separator").getBytes(Charset.defaultCharset()).length;
+ final int bytesPerLine = 0 + IPerfTestRunner.THROUGHPUT_MSG.getBytes().length;
+ final int bytesWritten = bytesPerLine * linesPerIteration * iterations;
+ final int threshold = 1073741824; // magic number: defined in perf9MMapLocation.xml
+
+ int todo = threshold - bytesWritten;
+ if (todo <= 0) {
+ return;
+ }
+ final byte[] filler = new byte[4096];
+ Arrays.fill(filler, (byte) 'X');
+ final String str = new String(filler, Charset.defaultCharset());
+ do {
+ runner.log(str);
+ } while ((todo -= (4096 + LINESEP)) > 0);
}
- private int runSingleThreadedTest(final IPerfTestRunner runner, final String name,
+ private int runSingleThreadedTest(final IPerfTestRunner runner, final int LINES, final String name,
final String resultFile) throws IOException {
final Histogram latency = createHistogram();
- final int LINES = throughput ? 50000 : 5000000;
runTest(runner, LINES, "end", latency, 1);
reportResult(resultFile, name, latency);
return LINES;
}
- static void reportResult(final String file, final String name, final Histogram histogram)
- throws IOException {
+ static void reportResult(final String file, final String name, final Histogram histogram) throws IOException {
final String result = createSamplingReport(name, histogram);
println(result);
@@ -167,8 +188,8 @@ public class PerfTest {
if (throughput) {
return data.getMax() + " operations/second";
}
- final String result = String.format(
- "avg=%.0f 99%%=%d 99.99%%=%d sampleCount=%d", data.getMean(), //
+ final String result = String.format("avg=%.0f 99%%=%d 99.99%%=%d sampleCount=%d", //
+ data.getMean(), //
data.getTwoNinesUpperBound(), //
data.getFourNinesUpperBound(), //
data.getCount() //
Re: [2/2] logging-log4j2 git commit: LOG4J2-928 added debug logging
to show duration of mmap, unmap and RAFile extend operations
Posted by Tomas Gradin <tg...@df.lth.se>.
> Hm... Git got confused for some reason... I only changed / added maybe
> 20 lines or so, certainly did not rewrite the whole file... Not sure why
> the diff shows up this way.
Line endings changed.
/Tomas G.
---------------------------------------------------------------------
To unsubscribe, e-mail: log4j-dev-unsubscribe@logging.apache.org
For additional commands, e-mail: log4j-dev-help@logging.apache.org
Re: [2/2] logging-log4j2 git commit: LOG4J2-928 added debug logging
to show duration of mmap, unmap and RAFile extend operations
Posted by Remko Popma <re...@gmail.com>.
Hm... Git got confused for some reason... I only changed / added maybe 20
lines or so, certainly did not rewrite the whole file... Not sure why the
diff shows up this way.
Lines I actually changed (line numbers of the new file):
126-134
147-155
168-175
182
198-199, 211-212
On Sat, Dec 27, 2014 at 10:29 PM, <rp...@apache.org> wrote:
> LOG4J2-928 added debug logging to show duration of mmap, unmap and
> RAFile extend operations
>
> Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
> Commit:
> http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/50278c35
> Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/50278c35
> Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/50278c35
>
> Branch: refs/heads/master
> Commit: 50278c35b5d4bc0607bc42591d1ef3e878d63329
> Parents: ae89596
> Author: rpopma <rp...@apache.org>
> Authored: Sat Dec 27 22:29:49 2014 +0900
> Committer: rpopma <rp...@apache.org>
> Committed: Sat Dec 27 22:29:49 2014 +0900
>
> ----------------------------------------------------------------------
> .../core/appender/MemoryMappedFileManager.java | 646 ++++++++++---------
> 1 file changed, 335 insertions(+), 311 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/50278c35/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
> ----------------------------------------------------------------------
> diff --git
> a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
> b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
> index cbb44cc..2eedb68 100644
> ---
> a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
> +++
> b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
> @@ -1,311 +1,335 @@
> -/*
> - * Licensed to the Apache Software Foundation (ASF) under one or more
> - * contributor license agreements. See the NOTICE file distributed with
> - * this work for additional information regarding copyright ownership.
> - * The ASF licenses this file to You under the Apache license, Version 2.0
> - * (the "License"); you may not use this file except in compliance with
> - * the License. You may obtain a copy of the License at
> - *
> - * http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> - * See the license for the specific language governing permissions and
> - * limitations under the license.
> - */
> -package org.apache.logging.log4j.core.appender;
> -
> -import java.io.File;
> -import java.io.IOException;
> -import java.io.OutputStream;
> -import java.io.RandomAccessFile;
> -import java.io.Serializable;
> -import java.lang.reflect.Method;
> -import java.nio.ByteOrder;
> -import java.nio.MappedByteBuffer;
> -import java.nio.channels.FileChannel;
> -import java.security.AccessController;
> -import java.security.PrivilegedActionException;
> -import java.security.PrivilegedExceptionAction;
> -import java.util.HashMap;
> -import java.util.Map;
> -
> -import org.apache.logging.log4j.core.Layout;
> -import org.apache.logging.log4j.core.util.Assert;
> -import org.apache.logging.log4j.core.util.Closer;
> -
> -/**
> - * Extends OutputStreamManager but instead of using a buffered output
> stream, this class maps a region of a file into
> - * memory and writes to this memory region.
> - * <p>
> - *
> - * @see <a href="
> http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java
> ">
> http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java
> </a>
> - * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">
> http://bugs.java.com/view_bug.do?bug_id=6893654</a>
> - * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">
> http://bugs.java.com/view_bug.do?bug_id=4724038</a>
> - * @see <a
> - * href="
> http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation
> ">
> http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation
> </a>
> - *
> - * @since 2.1
> - */
> -public class MemoryMappedFileManager extends OutputStreamManager {
> - static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
> - private static final MemoryMappedFileManagerFactory FACTORY = new
> MemoryMappedFileManagerFactory();
> -
> - private final boolean isForce;
> - private final int regionLength;
> - private final String advertiseURI;
> - private final RandomAccessFile randomAccessFile;
> - private final ThreadLocal<Boolean> isEndOfBatch = new
> ThreadLocal<Boolean>();
> - private MappedByteBuffer mappedBuffer;
> - private long mappingOffset;
> -
> - protected MemoryMappedFileManager(final RandomAccessFile file, final
> String fileName, final OutputStream os,
> - final boolean force, final long position, final int
> regionLength, final String advertiseURI,
> - final Layout<? extends Serializable> layout) throws
> IOException {
> - super(os, fileName, layout);
> - this.isForce = force;
> - this.randomAccessFile = Assert.requireNonNull(file,
> "RandomAccessFile");
> - this.regionLength = regionLength;
> - this.advertiseURI = advertiseURI;
> - this.isEndOfBatch.set(Boolean.FALSE);
> - this.mappedBuffer = mmap(randomAccessFile.getChannel(), position,
> regionLength);
> - this.mappingOffset = position;
> - }
> -
> - /**
> - * Returns the MemoryMappedFileManager.
> - *
> - * @param fileName The name of the file to manage.
> - * @param append true if the file should be appended to, false if it
> should be overwritten.
> - * @param isForce true if the contents should be flushed to disk on
> every write
> - * @param regionLength The mapped region length.
> - * @param advertiseURI the URI to use when advertising the file
> - * @param layout The layout.
> - * @return A MemoryMappedFileManager for the File.
> - */
> - public static MemoryMappedFileManager getFileManager(final String
> fileName, final boolean append,
> - final boolean isForce, final int regionLength, final String
> advertiseURI,
> - final Layout<? extends Serializable> layout) {
> - return (MemoryMappedFileManager) getManager(fileName, new
> FactoryData(append, isForce, regionLength,
> - advertiseURI, layout), FACTORY);
> - }
> -
> - public Boolean isEndOfBatch() {
> - return isEndOfBatch.get();
> - }
> -
> - public void setEndOfBatch(final boolean isEndOfBatch) {
> - this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
> - }
> -
> - @Override
> - protected synchronized void write(final byte[] bytes, int offset, int
> length) {
> - super.write(bytes, offset, length); // writes to dummy output
> stream
> -
> - while (length > mappedBuffer.remaining()) {
> - final int chunk = mappedBuffer.remaining();
> - mappedBuffer.put(bytes, offset, chunk);
> - offset += chunk;
> - length -= chunk;
> - remap();
> - }
> - mappedBuffer.put(bytes, offset, length);
> -
> - // no need to call flush() if force is true,
> - // already done in AbstractOutputStreamAppender.append
> - }
> -
> - private synchronized void remap() {
> - final long offset = this.mappingOffset + mappedBuffer.position();
> - final int length = mappedBuffer.remaining() + regionLength;
> - try {
> - unsafeUnmap(mappedBuffer);
> - final long fileLength = randomAccessFile.length() +
> regionLength;
> - randomAccessFile.setLength(fileLength);
> - mappedBuffer = mmap(randomAccessFile.getChannel(), offset,
> length);
> - mappingOffset = offset;
> - } catch (final Exception ex) {
> - LOGGER.error("Unable to remap " + getName() + ". " + ex);
> - }
> - }
> -
> - @Override
> - public synchronized void flush() {
> - mappedBuffer.force();
> - }
> -
> - @Override
> - public synchronized void close() {
> - final long length = mappingOffset + mappedBuffer.position();
> - try {
> - unsafeUnmap(mappedBuffer);
> - } catch (final Exception ex) {
> - LOGGER.error("Unable to unmap MappedBuffer " + getName() + ".
> " + ex);
> - }
> - try {
> - randomAccessFile.setLength(length);
> - randomAccessFile.close();
> - } catch (final IOException ex) {
> - LOGGER.error("Unable to close MemoryMappedFile " + getName()
> + ". " + ex);
> - }
> - }
> -
> - public static MappedByteBuffer mmap(final FileChannel fileChannel,
> final long start, final int size) throws IOException {
> - for (int i = 1;; i++) {
> - try {
> - final MappedByteBuffer map =
> fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
> - map.order(ByteOrder.nativeOrder());
> - return map;
> - } catch (final IOException e) {
> - if (e.getMessage() == null ||
> !e.getMessage().endsWith("user-mapped section open")) {
> - throw e;
> - }
> - if (i < 10) {
> - Thread.yield();
> - } else {
> - try {
> - Thread.sleep(1);
> - } catch (final InterruptedException ignored) {
> - Thread.currentThread().interrupt();
> - throw e;
> - }
> - }
> - }
> - }
> - }
> -
> - private static void unsafeUnmap(final MappedByteBuffer mbb) throws
> PrivilegedActionException {
> - AccessController.doPrivileged(new
> PrivilegedExceptionAction<Object>() {
> - @Override
> - public Object run() throws Exception {
> - final Method getCleanerMethod =
> mbb.getClass().getMethod("cleaner");
> - getCleanerMethod.setAccessible(true);
> - final Object cleaner = getCleanerMethod.invoke(mbb); //
> sun.misc.Cleaner instance
> - final Method cleanMethod =
> cleaner.getClass().getMethod("clean");
> - cleanMethod.invoke(cleaner);
> - return null;
> - }
> - });
> - }
> -
> - /**
> - * Returns the name of the File being managed.
> - *
> - * @return The name of the File being managed.
> - */
> - public String getFileName() {
> - return getName();
> - }
> -
> - /**
> - * Returns the length of the memory mapped region.
> - *
> - * @return the length of the mapped region
> - */
> - public int getRegionLength() {
> - return regionLength;
> - }
> -
> - /**
> - * Returns {@code true} if the content of the buffer should be forced
> to the storage device on every write,
> - * {@code false} otherwise.
> - * @return whether each write should be force-sync'ed
> - */
> - public boolean isImmediateFlush() {
> - return isForce;
> - }
> -
> - /** {@code OutputStream} subclass that does not write anything. */
> - static class DummyOutputStream extends OutputStream {
> - @Override
> - public void write(final int b) throws IOException {
> - }
> -
> - @Override
> - public void write(final byte[] b, final int off, final int len)
> throws IOException {
> - }
> - }
> -
> - /**
> - * Gets this FileManager's content format specified by:
> - * <p>
> - * Key: "fileURI" Value: provided "advertiseURI" param.
> - * </p>
> - *
> - * @return Map of content format keys supporting FileManager
> - */
> - @Override
> - public Map<String, String> getContentFormat() {
> - final Map<String, String> result = new HashMap<String,
> String>(super.getContentFormat());
> - result.put("fileURI", advertiseURI);
> - return result;
> - }
> -
> - /**
> - * Factory Data.
> - */
> - private static class FactoryData {
> - private final boolean append;
> - private final boolean force;
> - private final int regionLength;
> - private final String advertiseURI;
> - private final Layout<? extends Serializable> layout;
> -
> - /**
> - * Constructor.
> - *
> - * @param append Append to existing file or truncate.
> - * @param force forces the memory content to be written to the
> storage device on every event
> - * @param regionLength length of the mapped region
> - */
> - public FactoryData(final boolean append, final boolean force,
> final int regionLength,
> - final String advertiseURI, final Layout<? extends
> Serializable> layout) {
> - this.append = append;
> - this.force = force;
> - this.regionLength = regionLength;
> - this.advertiseURI = advertiseURI;
> - this.layout = layout;
> - }
> - }
> -
> - /**
> - * Factory to create a MemoryMappedFileManager.
> - */
> - private static class MemoryMappedFileManagerFactory implements
> ManagerFactory<MemoryMappedFileManager, FactoryData> {
> -
> - /**
> - * Create a MemoryMappedFileManager.
> - *
> - * @param name The name of the File.
> - * @param data The FactoryData
> - * @return The MemoryMappedFileManager for the File.
> - */
> - @SuppressWarnings("resource")
> - @Override
> - public MemoryMappedFileManager createManager(final String name,
> final FactoryData data) {
> - final File file = new File(name);
> - final File parent = file.getParentFile();
> - if (null != parent && !parent.exists()) {
> - parent.mkdirs();
> - }
> - if (!data.append) {
> - file.delete();
> - }
> -
> - final OutputStream os = new DummyOutputStream();
> - RandomAccessFile raf = null;
> - try {
> - raf = new RandomAccessFile(name, "rw");
> - final long position = (data.append) ? raf.length() : 0;
> - raf.setLength(position + data.regionLength);
> - return new MemoryMappedFileManager(raf, name, os,
> data.force, position, data.regionLength,
> - data.advertiseURI, data.layout);
> - } catch (final Exception ex) {
> - LOGGER.error("MemoryMappedFileManager (" + name + ") " +
> ex);
> - Closer.closeSilently(raf);
> - }
> - return null;
> - }
> - }
> -}
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache license, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the license for the specific language governing permissions and
> + * limitations under the license.
> + */
> +package org.apache.logging.log4j.core.appender;
> +
> +import java.io.File;
> +import java.io.IOException;
> +import java.io.OutputStream;
> +import java.io.RandomAccessFile;
> +import java.io.Serializable;
> +import java.lang.reflect.Method;
> +import java.nio.ByteOrder;
> +import java.nio.MappedByteBuffer;
> +import java.nio.channels.FileChannel;
> +import java.security.AccessController;
> +import java.security.PrivilegedActionException;
> +import java.security.PrivilegedExceptionAction;
> +import java.util.HashMap;
> +import java.util.Map;
> +
> +import org.apache.logging.log4j.core.Layout;
> +import org.apache.logging.log4j.core.util.Assert;
> +import org.apache.logging.log4j.core.util.Closer;
> +
> +/**
> + * Extends OutputStreamManager but instead of using a buffered output
> stream, this class maps a region of a file into
> + * memory and writes to this memory region.
> + * <p>
> + *
> + * @see <a
> + * href="
> http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java
> ">
> http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java
> </a>
> + * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">
> http://bugs.java.com/view_bug.do?bug_id=6893654</a>
> + * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">
> http://bugs.java.com/view_bug.do?bug_id=4724038</a>
> + * @see <a
> + * href="
> http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation
> ">
> http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation
> </a>
> + *
> + * @since 2.1
> + */
> +public class MemoryMappedFileManager extends OutputStreamManager {
> + static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
> + private static final MemoryMappedFileManagerFactory FACTORY = new
> MemoryMappedFileManagerFactory();
> +
> + private final boolean isForce;
> + private final int regionLength;
> + private final String advertiseURI;
> + private final RandomAccessFile randomAccessFile;
> + private final ThreadLocal<Boolean> isEndOfBatch = new
> ThreadLocal<Boolean>();
> + private MappedByteBuffer mappedBuffer;
> + private long mappingOffset;
> +
> + protected MemoryMappedFileManager(final RandomAccessFile file, final
> String fileName, final OutputStream os,
> + final boolean force, final long position, final int
> regionLength, final String advertiseURI,
> + final Layout<? extends Serializable> layout) throws
> IOException {
> + super(os, fileName, layout);
> + this.isForce = force;
> + this.randomAccessFile = Assert.requireNonNull(file,
> "RandomAccessFile");
> + this.regionLength = regionLength;
> + this.advertiseURI = advertiseURI;
> + this.isEndOfBatch.set(Boolean.FALSE);
> + this.mappedBuffer = mmap(randomAccessFile.getChannel(),
> getFileName(), position, regionLength);
> + this.mappingOffset = position;
> + }
> +
> + /**
> + * Returns the MemoryMappedFileManager.
> + *
> + * @param fileName The name of the file to manage.
> + * @param append true if the file should be appended to, false if it
> should be overwritten.
> + * @param isForce true if the contents should be flushed to disk on
> every write
> + * @param regionLength The mapped region length.
> + * @param advertiseURI the URI to use when advertising the file
> + * @param layout The layout.
> + * @return A MemoryMappedFileManager for the File.
> + */
> + public static MemoryMappedFileManager getFileManager(final String
> fileName, final boolean append,
> + final boolean isForce, final int regionLength, final String
> advertiseURI,
> + final Layout<? extends Serializable> layout) {
> + return (MemoryMappedFileManager) getManager(fileName, new
> FactoryData(append, isForce, regionLength,
> + advertiseURI, layout), FACTORY);
> + }
> +
> + public Boolean isEndOfBatch() {
> + return isEndOfBatch.get();
> + }
> +
> + public void setEndOfBatch(final boolean isEndOfBatch) {
> + this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
> + }
> +
> + @Override
> + protected synchronized void write(final byte[] bytes, int offset, int
> length) {
> + super.write(bytes, offset, length); // writes to dummy output
> stream
> +
> + while (length > mappedBuffer.remaining()) {
> + final int chunk = mappedBuffer.remaining();
> + mappedBuffer.put(bytes, offset, chunk);
> + offset += chunk;
> + length -= chunk;
> + remap();
> + }
> + mappedBuffer.put(bytes, offset, length);
> +
> + // no need to call flush() if force is true,
> + // already done in AbstractOutputStreamAppender.append
> + }
> +
> + private synchronized void remap() {
> + final long offset = this.mappingOffset + mappedBuffer.position();
> + final int length = mappedBuffer.remaining() + regionLength;
> + try {
> + unsafeUnmap(mappedBuffer);
> + final long fileLength = randomAccessFile.length() +
> regionLength;
> + LOGGER.debug("MMapAppender extending {} by {} bytes to {}",
> getFileName(), regionLength, fileLength);
> +
> + long startNanos = System.nanoTime();
> + randomAccessFile.setLength(fileLength);
> + final float millis = (float) ((System.nanoTime() -
> startNanos) / (1000.0 * 1000.0));
> + LOGGER.debug("MMapAppender extended {} OK in {} millis",
> getFileName(), millis);
> +
> + mappedBuffer = mmap(randomAccessFile.getChannel(),
> getFileName(), offset, length);
> + mappingOffset = offset;
> + } catch (final Exception ex) {
> + LOGGER.error("Unable to remap " + getName() + ". " + ex);
> + }
> + }
> +
> + @Override
> + public synchronized void flush() {
> + mappedBuffer.force();
> + }
> +
> + @Override
> + public synchronized void close() {
> + final long position = mappedBuffer.position();
> + final long length = mappingOffset + position;
> + try {
> + unsafeUnmap(mappedBuffer);
> + } catch (final Exception ex) {
> + LOGGER.error("Unable to unmap MappedBuffer " + getName() + ".
> " + ex);
> + }
> + try {
> + LOGGER.debug("MMapAppender closing. Setting {} length to {}
> (offset {} + position {})", getFileName(),
> + length, mappingOffset, position);
> + randomAccessFile.setLength(length);
> + randomAccessFile.close();
> + } catch (final IOException ex) {
> + LOGGER.error("Unable to close MemoryMappedFile " + getName()
> + ". " + ex);
> + }
> + }
> +
> + public static MappedByteBuffer mmap(final FileChannel fileChannel,
> final String fileName, final long start,
> + final int size) throws IOException {
> + for (int i = 1;; i++) {
> + try {
> + LOGGER.debug("MMapAppender remapping {} start={},
> size={}", fileName, start, size);
> +
> + final long startNanos = System.nanoTime();
> + final MappedByteBuffer map =
> fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
> + map.order(ByteOrder.nativeOrder());
> +
> + final float millis = (float) ((System.nanoTime() -
> startNanos) / (1000.0 * 1000.0));
> + LOGGER.debug("MMapAppender remapped {} OK in {} millis",
> fileName, millis);
> +
> + return map;
> + } catch (final IOException e) {
> + if (e.getMessage() == null ||
> !e.getMessage().endsWith("user-mapped section open")) {
> + throw e;
> + }
> + LOGGER.debug("Remap attempt {}/10 failed. Retrying...",
> i, e);
> + if (i < 10) {
> + Thread.yield();
> + } else {
> + try {
> + Thread.sleep(1);
> + } catch (final InterruptedException ignored) {
> + Thread.currentThread().interrupt();
> + throw e;
> + }
> + }
> + }
> + }
> + }
> +
> + private static void unsafeUnmap(final MappedByteBuffer mbb) throws
> PrivilegedActionException {
> + LOGGER.debug("MMapAppender unmapping old buffer...");
> + final long startNanos = System.nanoTime();
> + AccessController.doPrivileged(new
> PrivilegedExceptionAction<Object>() {
> + @Override
> + public Object run() throws Exception {
> + final Method getCleanerMethod =
> mbb.getClass().getMethod("cleaner");
> + getCleanerMethod.setAccessible(true);
> + final Object cleaner = getCleanerMethod.invoke(mbb); //
> sun.misc.Cleaner instance
> + final Method cleanMethod =
> cleaner.getClass().getMethod("clean");
> + cleanMethod.invoke(cleaner);
> + return null;
> + }
> + });
> + final float millis = (float) ((System.nanoTime() - startNanos) /
> (1000.0 * 1000.0));
> + LOGGER.debug("MMapAppender unmapped buffer OK in {} millis",
> millis);
> + }
> +
> + /**
> + * Returns the name of the File being managed.
> + *
> + * @return The name of the File being managed.
> + */
> + public String getFileName() {
> + return getName();
> + }
> +
> + /**
> + * Returns the length of the memory mapped region.
> + *
> + * @return the length of the mapped region
> + */
> + public int getRegionLength() {
> + return regionLength;
> + }
> +
> + /**
> + * Returns {@code true} if the content of the buffer should be forced
> to the storage device on every write,
> + * {@code false} otherwise.
> + *
> + * @return whether each write should be force-sync'ed
> + */
> + public boolean isImmediateFlush() {
> + return isForce;
> + }
> +
> + /** {@code OutputStream} subclass that does not write anything. */
> + static class DummyOutputStream extends OutputStream {
> + @Override
> + public void write(final int b) throws IOException {
> + }
> +
> + @Override
> + public void write(final byte[] b, final int off, final int len)
> throws IOException {
> + }
> + }
> +
> + /**
> + * Gets this FileManager's content format specified by:
> + * <p>
> + * Key: "fileURI" Value: provided "advertiseURI" param.
> + * </p>
> + *
> + * @return Map of content format keys supporting FileManager
> + */
> + @Override
> + public Map<String, String> getContentFormat() {
> + final Map<String, String> result = new HashMap<String,
> String>(super.getContentFormat());
> + result.put("fileURI", advertiseURI);
> + return result;
> + }
> +
> + /**
> + * Factory Data.
> + */
> + private static class FactoryData {
> + private final boolean append;
> + private final boolean force;
> + private final int regionLength;
> + private final String advertiseURI;
> + private final Layout<? extends Serializable> layout;
> +
> + /**
> + * Constructor.
> + *
> + * @param append Append to existing file or truncate.
> + * @param force forces the memory content to be written to the
> storage device on every event
> + * @param regionLength length of the mapped region
> + */
> + public FactoryData(final boolean append, final boolean force,
> final int regionLength,
> + final String advertiseURI, final Layout<? extends
> Serializable> layout) {
> + this.append = append;
> + this.force = force;
> + this.regionLength = regionLength;
> + this.advertiseURI = advertiseURI;
> + this.layout = layout;
> + }
> + }
> +
> + /**
> + * Factory to create a MemoryMappedFileManager.
> + */
> + private static class MemoryMappedFileManagerFactory implements
> ManagerFactory<MemoryMappedFileManager, FactoryData> {
> +
> + /**
> + * Create a MemoryMappedFileManager.
> + *
> + * @param name The name of the File.
> + * @param data The FactoryData
> + * @return The MemoryMappedFileManager for the File.
> + */
> + @SuppressWarnings("resource")
> + @Override
> + public MemoryMappedFileManager createManager(final String name,
> final FactoryData data) {
> + final File file = new File(name);
> + final File parent = file.getParentFile();
> + if (null != parent && !parent.exists()) {
> + parent.mkdirs();
> + }
> + if (!data.append) {
> + file.delete();
> + }
> +
> + final OutputStream os = new DummyOutputStream();
> + RandomAccessFile raf = null;
> + try {
> + raf = new RandomAccessFile(name, "rw");
> + final long position = (data.append) ? raf.length() : 0;
> + raf.setLength(position + data.regionLength);
> + return new MemoryMappedFileManager(raf, name, os,
> data.force, position, data.regionLength,
> + data.advertiseURI, data.layout);
> + } catch (final Exception ex) {
> + LOGGER.error("MemoryMappedFileManager (" + name + ") " +
> ex);
> + Closer.closeSilently(raf);
> + }
> + return null;
> + }
> + }
> +}
>
>
[2/2] logging-log4j2 git commit: LOG4J2-928 added debug logging to
show duration of mmap, unmap and RAFile extend operations
Posted by rp...@apache.org.
LOG4J2-928 added debug logging to show duration of mmap, unmap and
RAFile extend operations
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/50278c35
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/50278c35
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/50278c35
Branch: refs/heads/master
Commit: 50278c35b5d4bc0607bc42591d1ef3e878d63329
Parents: ae89596
Author: rpopma <rp...@apache.org>
Authored: Sat Dec 27 22:29:49 2014 +0900
Committer: rpopma <rp...@apache.org>
Committed: Sat Dec 27 22:29:49 2014 +0900
----------------------------------------------------------------------
.../core/appender/MemoryMappedFileManager.java | 646 ++++++++++---------
1 file changed, 335 insertions(+), 311 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/50278c35/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
index cbb44cc..2eedb68 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
@@ -1,311 +1,335 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache license, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the license for the specific language governing permissions and
- * limitations under the license.
- */
-package org.apache.logging.log4j.core.appender;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.RandomAccessFile;
-import java.io.Serializable;
-import java.lang.reflect.Method;
-import java.nio.ByteOrder;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.security.AccessController;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.logging.log4j.core.Layout;
-import org.apache.logging.log4j.core.util.Assert;
-import org.apache.logging.log4j.core.util.Closer;
-
-/**
- * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
- * memory and writes to this memory region.
- * <p>
- *
- * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
- * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
- * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
- * @see <a
- * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
- *
- * @since 2.1
- */
-public class MemoryMappedFileManager extends OutputStreamManager {
- static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
- private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
-
- private final boolean isForce;
- private final int regionLength;
- private final String advertiseURI;
- private final RandomAccessFile randomAccessFile;
- private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
- private MappedByteBuffer mappedBuffer;
- private long mappingOffset;
-
- protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
- final boolean force, final long position, final int regionLength, final String advertiseURI,
- final Layout<? extends Serializable> layout) throws IOException {
- super(os, fileName, layout);
- this.isForce = force;
- this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
- this.regionLength = regionLength;
- this.advertiseURI = advertiseURI;
- this.isEndOfBatch.set(Boolean.FALSE);
- this.mappedBuffer = mmap(randomAccessFile.getChannel(), position, regionLength);
- this.mappingOffset = position;
- }
-
- /**
- * Returns the MemoryMappedFileManager.
- *
- * @param fileName The name of the file to manage.
- * @param append true if the file should be appended to, false if it should be overwritten.
- * @param isForce true if the contents should be flushed to disk on every write
- * @param regionLength The mapped region length.
- * @param advertiseURI the URI to use when advertising the file
- * @param layout The layout.
- * @return A MemoryMappedFileManager for the File.
- */
- public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
- final boolean isForce, final int regionLength, final String advertiseURI,
- final Layout<? extends Serializable> layout) {
- return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
- advertiseURI, layout), FACTORY);
- }
-
- public Boolean isEndOfBatch() {
- return isEndOfBatch.get();
- }
-
- public void setEndOfBatch(final boolean isEndOfBatch) {
- this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
- }
-
- @Override
- protected synchronized void write(final byte[] bytes, int offset, int length) {
- super.write(bytes, offset, length); // writes to dummy output stream
-
- while (length > mappedBuffer.remaining()) {
- final int chunk = mappedBuffer.remaining();
- mappedBuffer.put(bytes, offset, chunk);
- offset += chunk;
- length -= chunk;
- remap();
- }
- mappedBuffer.put(bytes, offset, length);
-
- // no need to call flush() if force is true,
- // already done in AbstractOutputStreamAppender.append
- }
-
- private synchronized void remap() {
- final long offset = this.mappingOffset + mappedBuffer.position();
- final int length = mappedBuffer.remaining() + regionLength;
- try {
- unsafeUnmap(mappedBuffer);
- final long fileLength = randomAccessFile.length() + regionLength;
- randomAccessFile.setLength(fileLength);
- mappedBuffer = mmap(randomAccessFile.getChannel(), offset, length);
- mappingOffset = offset;
- } catch (final Exception ex) {
- LOGGER.error("Unable to remap " + getName() + ". " + ex);
- }
- }
-
- @Override
- public synchronized void flush() {
- mappedBuffer.force();
- }
-
- @Override
- public synchronized void close() {
- final long length = mappingOffset + mappedBuffer.position();
- try {
- unsafeUnmap(mappedBuffer);
- } catch (final Exception ex) {
- LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
- }
- try {
- randomAccessFile.setLength(length);
- randomAccessFile.close();
- } catch (final IOException ex) {
- LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
- }
- }
-
- public static MappedByteBuffer mmap(final FileChannel fileChannel, final long start, final int size) throws IOException {
- for (int i = 1;; i++) {
- try {
- final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
- map.order(ByteOrder.nativeOrder());
- return map;
- } catch (final IOException e) {
- if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
- throw e;
- }
- if (i < 10) {
- Thread.yield();
- } else {
- try {
- Thread.sleep(1);
- } catch (final InterruptedException ignored) {
- Thread.currentThread().interrupt();
- throw e;
- }
- }
- }
- }
- }
-
- private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
- AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
- getCleanerMethod.setAccessible(true);
- final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
- final Method cleanMethod = cleaner.getClass().getMethod("clean");
- cleanMethod.invoke(cleaner);
- return null;
- }
- });
- }
-
- /**
- * Returns the name of the File being managed.
- *
- * @return The name of the File being managed.
- */
- public String getFileName() {
- return getName();
- }
-
- /**
- * Returns the length of the memory mapped region.
- *
- * @return the length of the mapped region
- */
- public int getRegionLength() {
- return regionLength;
- }
-
- /**
- * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
- * {@code false} otherwise.
- * @return whether each write should be force-sync'ed
- */
- public boolean isImmediateFlush() {
- return isForce;
- }
-
- /** {@code OutputStream} subclass that does not write anything. */
- static class DummyOutputStream extends OutputStream {
- @Override
- public void write(final int b) throws IOException {
- }
-
- @Override
- public void write(final byte[] b, final int off, final int len) throws IOException {
- }
- }
-
- /**
- * Gets this FileManager's content format specified by:
- * <p>
- * Key: "fileURI" Value: provided "advertiseURI" param.
- * </p>
- *
- * @return Map of content format keys supporting FileManager
- */
- @Override
- public Map<String, String> getContentFormat() {
- final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
- result.put("fileURI", advertiseURI);
- return result;
- }
-
- /**
- * Factory Data.
- */
- private static class FactoryData {
- private final boolean append;
- private final boolean force;
- private final int regionLength;
- private final String advertiseURI;
- private final Layout<? extends Serializable> layout;
-
- /**
- * Constructor.
- *
- * @param append Append to existing file or truncate.
- * @param force forces the memory content to be written to the storage device on every event
- * @param regionLength length of the mapped region
- */
- public FactoryData(final boolean append, final boolean force, final int regionLength,
- final String advertiseURI, final Layout<? extends Serializable> layout) {
- this.append = append;
- this.force = force;
- this.regionLength = regionLength;
- this.advertiseURI = advertiseURI;
- this.layout = layout;
- }
- }
-
- /**
- * Factory to create a MemoryMappedFileManager.
- */
- private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
-
- /**
- * Create a MemoryMappedFileManager.
- *
- * @param name The name of the File.
- * @param data The FactoryData
- * @return The MemoryMappedFileManager for the File.
- */
- @SuppressWarnings("resource")
- @Override
- public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
- final File file = new File(name);
- final File parent = file.getParentFile();
- if (null != parent && !parent.exists()) {
- parent.mkdirs();
- }
- if (!data.append) {
- file.delete();
- }
-
- final OutputStream os = new DummyOutputStream();
- RandomAccessFile raf = null;
- try {
- raf = new RandomAccessFile(name, "rw");
- final long position = (data.append) ? raf.length() : 0;
- raf.setLength(position + data.regionLength);
- return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
- data.advertiseURI, data.layout);
- } catch (final Exception ex) {
- LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
- Closer.closeSilently(raf);
- }
- return null;
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.appender;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.io.Serializable;
+import java.lang.reflect.Method;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.util.Assert;
+import org.apache.logging.log4j.core.util.Closer;
+
+/**
+ * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
+ * memory and writes to this memory region.
+ * <p>
+ *
+ * @see <a
+ * href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
+ * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
+ * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
+ * @see <a
+ * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
+ *
+ * @since 2.1
+ */
+public class MemoryMappedFileManager extends OutputStreamManager {
+ static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
+ private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
+
+ private final boolean isForce;
+ private final int regionLength;
+ private final String advertiseURI;
+ private final RandomAccessFile randomAccessFile;
+ private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
+ private MappedByteBuffer mappedBuffer;
+ private long mappingOffset;
+
+ protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
+ final boolean force, final long position, final int regionLength, final String advertiseURI,
+ final Layout<? extends Serializable> layout) throws IOException {
+ super(os, fileName, layout);
+ this.isForce = force;
+ this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
+ this.regionLength = regionLength;
+ this.advertiseURI = advertiseURI;
+ this.isEndOfBatch.set(Boolean.FALSE);
+ this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
+ this.mappingOffset = position;
+ }
+
+ /**
+ * Returns the MemoryMappedFileManager.
+ *
+ * @param fileName The name of the file to manage.
+ * @param append true if the file should be appended to, false if it should be overwritten.
+ * @param isForce true if the contents should be flushed to disk on every write
+ * @param regionLength The mapped region length.
+ * @param advertiseURI the URI to use when advertising the file
+ * @param layout The layout.
+ * @return A MemoryMappedFileManager for the File.
+ */
+ public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
+ final boolean isForce, final int regionLength, final String advertiseURI,
+ final Layout<? extends Serializable> layout) {
+ return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
+ advertiseURI, layout), FACTORY);
+ }
+
+ public Boolean isEndOfBatch() {
+ return isEndOfBatch.get();
+ }
+
+ public void setEndOfBatch(final boolean isEndOfBatch) {
+ this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
+ }
+
+ @Override
+ protected synchronized void write(final byte[] bytes, int offset, int length) {
+ super.write(bytes, offset, length); // writes to dummy output stream
+
+ while (length > mappedBuffer.remaining()) {
+ final int chunk = mappedBuffer.remaining();
+ mappedBuffer.put(bytes, offset, chunk);
+ offset += chunk;
+ length -= chunk;
+ remap();
+ }
+ mappedBuffer.put(bytes, offset, length);
+
+ // no need to call flush() if force is true,
+ // already done in AbstractOutputStreamAppender.append
+ }
+
+ private synchronized void remap() {
+ final long offset = this.mappingOffset + mappedBuffer.position();
+ final int length = mappedBuffer.remaining() + regionLength;
+ try {
+ unsafeUnmap(mappedBuffer);
+ final long fileLength = randomAccessFile.length() + regionLength;
+ LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
+
+ long startNanos = System.nanoTime();
+ randomAccessFile.setLength(fileLength);
+ final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
+ LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
+
+ mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
+ mappingOffset = offset;
+ } catch (final Exception ex) {
+ LOGGER.error("Unable to remap " + getName() + ". " + ex);
+ }
+ }
+
+ @Override
+ public synchronized void flush() {
+ mappedBuffer.force();
+ }
+
+ @Override
+ public synchronized void close() {
+ final long position = mappedBuffer.position();
+ final long length = mappingOffset + position;
+ try {
+ unsafeUnmap(mappedBuffer);
+ } catch (final Exception ex) {
+ LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
+ }
+ try {
+ LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
+ length, mappingOffset, position);
+ randomAccessFile.setLength(length);
+ randomAccessFile.close();
+ } catch (final IOException ex) {
+ LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
+ }
+ }
+
+ public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
+ final int size) throws IOException {
+ for (int i = 1;; i++) {
+ try {
+ LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
+
+ final long startNanos = System.nanoTime();
+ final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
+ map.order(ByteOrder.nativeOrder());
+
+ final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
+ LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
+
+ return map;
+ } catch (final IOException e) {
+ if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
+ throw e;
+ }
+ LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
+ if (i < 10) {
+ Thread.yield();
+ } else {
+ try {
+ Thread.sleep(1);
+ } catch (final InterruptedException ignored) {
+ Thread.currentThread().interrupt();
+ throw e;
+ }
+ }
+ }
+ }
+ }
+
+ private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
+ LOGGER.debug("MMapAppender unmapping old buffer...");
+ final long startNanos = System.nanoTime();
+ AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
+ getCleanerMethod.setAccessible(true);
+ final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
+ final Method cleanMethod = cleaner.getClass().getMethod("clean");
+ cleanMethod.invoke(cleaner);
+ return null;
+ }
+ });
+ final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
+ LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
+ }
+
+ /**
+ * Returns the name of the File being managed.
+ *
+ * @return The name of the File being managed.
+ */
+ public String getFileName() {
+ return getName();
+ }
+
+ /**
+ * Returns the length of the memory mapped region.
+ *
+ * @return the length of the mapped region
+ */
+ public int getRegionLength() {
+ return regionLength;
+ }
+
+ /**
+ * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
+ * {@code false} otherwise.
+ *
+ * @return whether each write should be force-sync'ed
+ */
+ public boolean isImmediateFlush() {
+ return isForce;
+ }
+
+ /** {@code OutputStream} subclass that does not write anything. */
+ static class DummyOutputStream extends OutputStream {
+ @Override
+ public void write(final int b) throws IOException {
+ }
+
+ @Override
+ public void write(final byte[] b, final int off, final int len) throws IOException {
+ }
+ }
+
+ /**
+ * Gets this FileManager's content format specified by:
+ * <p>
+ * Key: "fileURI" Value: provided "advertiseURI" param.
+ * </p>
+ *
+ * @return Map of content format keys supporting FileManager
+ */
+ @Override
+ public Map<String, String> getContentFormat() {
+ final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
+ result.put("fileURI", advertiseURI);
+ return result;
+ }
+
+ /**
+ * Factory Data.
+ */
+ private static class FactoryData {
+ private final boolean append;
+ private final boolean force;
+ private final int regionLength;
+ private final String advertiseURI;
+ private final Layout<? extends Serializable> layout;
+
+ /**
+ * Constructor.
+ *
+ * @param append Append to existing file or truncate.
+ * @param force forces the memory content to be written to the storage device on every event
+ * @param regionLength length of the mapped region
+ */
+ public FactoryData(final boolean append, final boolean force, final int regionLength,
+ final String advertiseURI, final Layout<? extends Serializable> layout) {
+ this.append = append;
+ this.force = force;
+ this.regionLength = regionLength;
+ this.advertiseURI = advertiseURI;
+ this.layout = layout;
+ }
+ }
+
+ /**
+ * Factory to create a MemoryMappedFileManager.
+ */
+ private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
+
+ /**
+ * Create a MemoryMappedFileManager.
+ *
+ * @param name The name of the File.
+ * @param data The FactoryData
+ * @return The MemoryMappedFileManager for the File.
+ */
+ @SuppressWarnings("resource")
+ @Override
+ public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
+ final File file = new File(name);
+ final File parent = file.getParentFile();
+ if (null != parent && !parent.exists()) {
+ parent.mkdirs();
+ }
+ if (!data.append) {
+ file.delete();
+ }
+
+ final OutputStream os = new DummyOutputStream();
+ RandomAccessFile raf = null;
+ try {
+ raf = new RandomAccessFile(name, "rw");
+ final long position = (data.append) ? raf.length() : 0;
+ raf.setLength(position + data.regionLength);
+ return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
+ data.advertiseURI, data.layout);
+ } catch (final Exception ex) {
+ LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
+ Closer.closeSilently(raf);
+ }
+ return null;
+ }
+ }
+}