You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@logging.apache.org by rp...@apache.org on 2014/09/18 18:24:50 UTC
[03/17] git commit: various improvements and fixes in appender and
manager
various improvements and fixes in appender and manager
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo
Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/e751c22a
Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/e751c22a
Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/e751c22a
Branch: refs/heads/master
Commit: e751c22af16e7b0c4a96932a9ede8ddf51426eb1
Parents: ebbe63a
Author: rpopma <rp...@apache.org>
Authored: Wed Sep 17 00:15:44 2014 +0900
Committer: rpopma <rp...@apache.org>
Committed: Wed Sep 17 00:15:44 2014 +0900
----------------------------------------------------------------------
.../core/appender/MemoryMappedFileAppender.java | 99 ++++++++++----------
.../core/appender/MemoryMappedFileManager.java | 41 +++++---
2 files changed, 78 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/e751c22a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileAppender.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileAppender.java
index f0910b5..785c3da 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileAppender.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileAppender.java
@@ -35,10 +35,10 @@ import org.apache.logging.log4j.core.util.Booleans;
import org.apache.logging.log4j.core.util.Integers;
/**
- * File Appender.
+ * Memory Mapped File Appender.
*/
@Plugin(name = "MemoryMappedFile", category = "Core", elementType = "appender", printObject = true)
-public final class MemoryMappedFileAppender extends AbstractOutputStreamAppender<RandomAccessFileManager> {
+public final class MemoryMappedFileAppender extends AbstractOutputStreamAppender<MemoryMappedFileManager> {
private static final long serialVersionUID = 1L;
@@ -46,13 +46,12 @@ public final class MemoryMappedFileAppender extends AbstractOutputStreamAppender
private Object advertisement;
private final Advertiser advertiser;
- private MemoryMappedFileAppender(final String name, final Layout<? extends Serializable> layout, final Filter filter,
- final RandomAccessFileManager manager, final String filename, final boolean ignoreExceptions,
- final boolean immediateFlush, final Advertiser advertiser) {
+ private MemoryMappedFileAppender(final String name, final Layout<? extends Serializable> layout,
+ final Filter filter, final MemoryMappedFileManager manager, final String filename,
+ final boolean ignoreExceptions, final boolean immediateFlush, final Advertiser advertiser) {
super(name, layout, filter, ignoreExceptions, immediateFlush, manager);
if (advertiser != null) {
- final Map<String, String> configuration = new HashMap<String, String>(
- layout.getContentFormat());
+ final Map<String, String> configuration = new HashMap<String, String>(layout.getContentFormat());
configuration.putAll(manager.getContentFormat());
configuration.put("contentType", layout.getContentType());
configuration.put("name", name);
@@ -96,82 +95,82 @@ public final class MemoryMappedFileAppender extends AbstractOutputStreamAppender
public String getFileName() {
return this.fileName;
}
-
+
/**
- * Returns the size of the file manager's buffer.
- * @return the buffer size
+ * Returns the length of the memory mapped region.
+ *
+ * @return the length of the memory mapped region
*/
- public int getBufferSize() {
- return getManager().getBufferSize();
+ public int getRegionLength() {
+ return getManager().getRegionLength();
}
- // difference from standard File Appender:
- // locking is not supported and buffering cannot be switched off
/**
- * Create a File Appender.
+ * Create a Memory Mapped File Appender.
*
* @param fileName The name and path of the file.
- * @param append "True" if the file should be appended to, "false" if it
- * should be overwritten. The default is "true".
+ * @param append "True" if the file should be appended to, "false" if it should be overwritten. The default is
+ * "true".
* @param name The name of the Appender.
- * @param immediateFlush "true" if the contents should be flushed on every
- * write, "false" otherwise. The default is "true".
- * @param bufferSizeStr The buffer size, defaults to {@value RandomAccessFileManager#DEFAULT_BUFFER_SIZE}.
- * @param ignore If {@code "true"} (default) exceptions encountered when appending events are logged; otherwise
- * they are propagated to the caller.
- * @param layout The layout to use to format the event. If no layout is
- * provided the default PatternLayout will be used.
+ * @param immediateFlush "true" if the contents should be flushed on every write, "false" otherwise. The default is
+ * "true".
+ * @param regionLengthStr The buffer size, defaults to {@value RandomAccessFileManager#DEFAULT_REGION=LENGTH}.
+ * @param ignore If {@code "true"} (default) exceptions encountered when appending events are logged; otherwise they
+ * are propagated to the caller.
+ * @param layout The layout to use to format the event. If no layout is provided the default PatternLayout will be
+ * used.
* @param filter The filter, if any, to use.
- * @param advertise "true" if the appender configuration should be
- * advertised, "false" otherwise.
- * @param advertiseURI The advertised URI which can be used to retrieve the
- * file contents.
+ * @param advertise "true" if the appender configuration should be advertised, "false" otherwise.
+ * @param advertiseURI The advertised URI which can be used to retrieve the file contents.
* @param config The Configuration.
* @return The FileAppender.
*/
@PluginFactory
public static MemoryMappedFileAppender createAppender(
- @PluginAttribute("fileName") final String fileName,
- @PluginAttribute("append") final String append,
- @PluginAttribute("name") final String name,
- @PluginAttribute("immediateFlush") final String immediateFlush,
- @PluginAttribute("bufferSize") final String bufferSizeStr,
- @PluginAttribute("ignoreExceptions") final String ignore,
- @PluginElement("Layout") Layout<? extends Serializable> layout,
- @PluginElement("Filter") final Filter filter,
- @PluginAttribute("advertise") final String advertise,
- @PluginAttribute("advertiseURI") final String advertiseURI,
+ // @formatter:off
+ @PluginAttribute("fileName") final String fileName, //
+ @PluginAttribute("append") final String append, //
+ @PluginAttribute("name") final String name, //
+ @PluginAttribute("immediateFlush") final String immediateFlush, //
+ @PluginAttribute("regionLength") final String regionLengthStr, //
+ @PluginAttribute("ignoreExceptions") final String ignore, //
+ @PluginElement("Layout") Layout<? extends Serializable> layout, //
+ @PluginElement("Filter") final Filter filter, //
+ @PluginAttribute("advertise") final String advertise, //
+ @PluginAttribute("advertiseURI") final String advertiseURI, //
@PluginConfiguration final Configuration config) {
+ // @formatter:on
final boolean isAppend = Booleans.parseBoolean(append, true);
- final boolean isFlush = Booleans.parseBoolean(immediateFlush, true);
+ final boolean isForce = Booleans.parseBoolean(immediateFlush, true);
final boolean ignoreExceptions = Booleans.parseBoolean(ignore, true);
final boolean isAdvertise = Boolean.parseBoolean(advertise);
- final int bufferSize = Integers.parseInt(bufferSizeStr, RandomAccessFileManager.DEFAULT_BUFFER_SIZE);
+ final int regionLength = Integers.parseInt(regionLengthStr, MemoryMappedFileManager.DEFAULT_REGION_LENGTH);
+ final int actualRegionLength = Integers.ceilingNextPowerOfTwo(regionLength);
+ if (regionLength != actualRegionLength) {
+ LOGGER.info("MemoryMappedAppender[{}] Rounded up region length from {} to next power of two: {}", name,
+ regionLength, actualRegionLength);
+ }
if (name == null) {
- LOGGER.error("No name provided for FileAppender");
+ LOGGER.error("No name provided for MemoryMappedFileAppender");
return null;
}
if (fileName == null) {
- LOGGER.error("No filename provided for FileAppender with name "
- + name);
+ LOGGER.error("No filename provided for MemoryMappedFileAppender with name " + name);
return null;
}
if (layout == null) {
layout = PatternLayout.createDefaultLayout();
}
- final RandomAccessFileManager manager = RandomAccessFileManager.getFileManager(
- fileName, isAppend, isFlush, bufferSize, advertiseURI, layout
- );
+ final MemoryMappedFileManager manager = MemoryMappedFileManager.getFileManager(fileName, isAppend, isForce,
+ actualRegionLength, advertiseURI, layout);
if (manager == null) {
return null;
}
- return new MemoryMappedFileAppender(
- name, layout, filter, manager, fileName, ignoreExceptions, isFlush,
- isAdvertise ? config.getAdvertiser() : null
- );
+ return new MemoryMappedFileAppender(name, layout, filter, manager, fileName, ignoreExceptions, isForce,
+ isAdvertise ? config.getAdvertiser() : null);
}
}
http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/e751c22a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
----------------------------------------------------------------------
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
index 2a66d03..29f8c54 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/MemoryMappedFileManager.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.util.Assert;
import org.apache.logging.log4j.core.util.Closer;
/**
@@ -39,14 +40,15 @@ import org.apache.logging.log4j.core.util.Closer;
* memory and writes to this memory region.
* <p>
*
+ * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
* @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
* @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
* @see <a
* href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
- * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
*
*/
public class MemoryMappedFileManager extends OutputStreamManager {
+ static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
private final boolean isForce;
@@ -55,17 +57,19 @@ public class MemoryMappedFileManager extends OutputStreamManager {
private final RandomAccessFile randomAccessFile;
private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
private MappedByteBuffer mappedBuffer;
+ private long mappingOffset;
protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
final boolean force, final long position, final int regionLength, final String advertiseURI,
final Layout<? extends Serializable> layout) throws IOException {
super(os, fileName, layout);
this.isForce = force;
- this.randomAccessFile = file;
+ this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
this.regionLength = regionLength;
this.advertiseURI = advertiseURI;
this.isEndOfBatch.set(Boolean.FALSE);
this.mappedBuffer = mmap(randomAccessFile.getChannel(), position, regionLength);
+ this.mappingOffset = position;
}
/**
@@ -73,16 +77,16 @@ public class MemoryMappedFileManager extends OutputStreamManager {
*
* @param fileName The name of the file to manage.
* @param append true if the file should be appended to, false if it should be overwritten.
- * @param isFlush true if the contents should be flushed to disk on every write
- * @param bufferSize The buffer size.
+ * @param isForce true if the contents should be flushed to disk on every write
+ * @param regionLength The mapped region length.
* @param advertiseURI the URI to use when advertising the file
* @param layout The layout.
* @return A MemoryMappedFileManager for the File.
*/
public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
- final boolean isFlush, final int bufferSize, final String advertiseURI,
+ final boolean isForce, final int regionLength, final String advertiseURI,
final Layout<? extends Serializable> layout) {
- return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isFlush, bufferSize,
+ return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
advertiseURI, layout), FACTORY);
}
@@ -98,7 +102,11 @@ public class MemoryMappedFileManager extends OutputStreamManager {
protected synchronized void write(final byte[] bytes, int offset, int length) {
super.write(bytes, offset, length); // writes to dummy output stream
- if (length > mappedBuffer.remaining()) {
+ while (length > mappedBuffer.remaining()) {
+ final int chunk = mappedBuffer.remaining();
+ mappedBuffer.put(bytes, offset, chunk);
+ offset += chunk;
+ length -= chunk;
remap();
}
mappedBuffer.put(bytes, offset, length);
@@ -108,13 +116,15 @@ public class MemoryMappedFileManager extends OutputStreamManager {
}
}
- private void remap() {
- final long position = mappedBuffer.position();
+ private synchronized void remap() {
+ final long offset = this.mappingOffset + mappedBuffer.position();
+ final int length = mappedBuffer.remaining() + regionLength;
try {
unsafeUnmap(mappedBuffer);
- final long newLength = randomAccessFile.length() + regionLength;
- randomAccessFile.setLength(newLength);
- mappedBuffer = mmap(randomAccessFile.getChannel(), position, regionLength);
+ final long fileLength = randomAccessFile.length() + regionLength;
+ randomAccessFile.setLength(fileLength);
+ mappedBuffer = mmap(randomAccessFile.getChannel(), offset, length);
+ mappingOffset = offset;
} catch (final Exception ex) {
LOGGER.error("Unable to remap " + getName() + ". " + ex);
}
@@ -127,7 +137,14 @@ public class MemoryMappedFileManager extends OutputStreamManager {
@Override
public synchronized void close() {
+ final long length = mappingOffset + mappedBuffer.position();
+ try {
+ unsafeUnmap(mappedBuffer);
+ } catch (Exception ex) {
+ LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
+ }
try {
+ randomAccessFile.setLength(length);
randomAccessFile.close();
} catch (final IOException ex) {
LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);