You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/07/13 18:35:57 UTC
[2/9] lucenenet git commit: BUG: Lucene.Net.Store: Fixed
MMapDirectory and FSDirectory concurrency using solution suggested by Vincent
Van Den Berghe http://apache.markmail.org/message/hafnuhq2ydhfjmi2. Fixes
LUCENENET-521 and closes LUCENENET-530.
BUG: Lucene.Net.Store: Fixed MMapDirectory and FSDirectory concurrency using solution suggested by Vincent Van Den Berghe http://apache.markmail.org/message/hafnuhq2ydhfjmi2. Fixes LUCENENET-521 and closes LUCENENET-530.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/c0e7a25c
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/c0e7a25c
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/c0e7a25c
Branch: refs/heads/master
Commit: c0e7a25c4a5901a4b6bea4f92088b5135c1f550a
Parents: bfcb895
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Thu Jul 13 23:45:23 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Thu Jul 13 23:45:23 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Store/FSDirectory.cs | 18 +++--
src/Lucene.Net/Store/MMapDirectory.cs | 101 +++++++++--------------------
2 files changed, 43 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c0e7a25c/src/Lucene.Net/Store/FSDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs
index 36cac51..19f8e14 100644
--- a/src/Lucene.Net/Store/FSDirectory.cs
+++ b/src/Lucene.Net/Store/FSDirectory.cs
@@ -362,10 +362,12 @@ namespace Lucene.Net.Store
ISet<string> toSync = new HashSet<string>(names);
toSync.IntersectWith(m_staleFiles);
- foreach (var name in toSync)
- {
- Fsync(name);
- }
+ // LUCENENET specific: Fsync breaks concurrency here.
+ // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+ //foreach (var name in toSync)
+ //{
+ // Fsync(name);
+ //}
// fsync the directory itsself, but only if there was any file fsynced before
// (otherwise it can happen that the directory does not yet exist)!
@@ -487,6 +489,14 @@ namespace Lucene.Net.Store
//Debug.Assert(size == 0);
}
+ // LUCENENET specific - file.Flush(flushToDisk: true) required in .NET for concurrency
+ // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+ public override void Flush()
+ {
+ base.Flush();
+ file.Flush(flushToDisk: true);
+ }
+
protected override void Dispose(bool disposing)
{
if (disposing)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c0e7a25c/src/Lucene.Net/Store/MMapDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/MMapDirectory.cs b/src/Lucene.Net/Store/MMapDirectory.cs
index 349b926..68eebf6 100644
--- a/src/Lucene.Net/Store/MMapDirectory.cs
+++ b/src/Lucene.Net/Store/MMapDirectory.cs
@@ -174,21 +174,7 @@ namespace Lucene.Net.Store
// indeed "release all resources".
public static readonly bool UNMAP_SUPPORTED = true;
- //static MMapDirectory()
- //{
- // bool v;
- // try
- // {
- // Type.GetType("sun.misc.Cleaner");
- // Type.GetType("java.nio.DirectByteBuffer").GetMethod("cleaner");
- // v = true;
- // }
- // catch (Exception)
- // {
- // v = false;
- // }
- // UNMAP_SUPPORTED = v;
- //}
+ // LUCENENET specific: No need for static constructor, since unmap is always supported
/// <summary>
/// This property enables the workaround for unmapping the buffers
@@ -235,7 +221,7 @@ namespace Lucene.Net.Store
EnsureOpen();
var file = new FileInfo(Path.Combine(Directory.FullName, name));
- var c = new FileStream(file.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
+ var c = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
return new MMapIndexInput(this, "MMapIndexInput(path=\"" + file + "\")", c);
}
@@ -317,52 +303,14 @@ namespace Lucene.Net.Store
/// </summary>
protected override void FreeBuffer(ByteBuffer buffer)
{
- // .NET port: this should free the memory mapped view accessor
+ // LUCENENET specific: this should free the memory mapped view accessor
var mmfbb = buffer as MemoryMappedFileByteBuffer;
if (mmfbb != null)
mmfbb.Dispose();
- //if (UseUnmapHack)
- //{
- // try
- // {
- // AccessController.doPrivileged(new PrivilegedExceptionActionAnonymousInnerClassHelper(this, buffer));
- // }
- // catch (PrivilegedActionException e)
- // {
- // System.IO.IOException ioe = new System.IO.IOException("unable to unmap the mapped buffer");
- // ioe.initCause(e.InnerException);
- // throw ioe;
- // }
- //}
+ // LUCENENET specific: no need for UnmapHack
}
-
-
- //private class PrivilegedExceptionActionAnonymousInnerClassHelper : PrivilegedExceptionAction<Void>
- //{
- // private readonly MMapIndexInput OuterInstance;
-
- // private ByteBuffer Buffer;
-
- // public PrivilegedExceptionActionAnonymousInnerClassHelper(MMapIndexInput outerInstance, ByteBuffer buffer)
- // {
- // this.OuterInstance = outerInstance;
- // this.Buffer = buffer;
- // }
-
- // public override void Run()
- // {
- // Method getCleanerMethod = Buffer.GetType().GetMethod("cleaner");
- // getCleanerMethod.Accessible = true;
- // object cleaner = getCleanerMethod.invoke(Buffer);
- // if (cleaner != null)
- // {
- // cleaner.GetType().GetMethod("clean").invoke(cleaner);
- // }
- // //return null;
- // }
- //}
}
/// <summary>
@@ -372,6 +320,13 @@ namespace Lucene.Net.Store
if (Number.URShift(length, chunkSizePower) >= int.MaxValue)
throw new ArgumentException("RandomAccessFile too big for chunk size: " + fc.ToString());
+ // LUCENENET specific: Return empty buffer if length is 0, rather than attempting to create a MemoryMappedFile.
+ // Part of a solution provided by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+ if (length == 0)
+ {
+ return new[] { ByteBuffer.Allocate(0).AsReadOnlyBuffer() };
+ }
+
long chunkSize = 1L << chunkSizePower;
// we always allocate one more buffer, the last one may be a 0 byte one
@@ -379,21 +334,18 @@ namespace Lucene.Net.Store
ByteBuffer[] buffers = new ByteBuffer[nrBuffers];
- /*
- public static MemoryMappedFile CreateFromFile(FileStream fileStream, String mapName, Int64 capacity,
- MemoryMappedFileAccess access, MemoryMappedFileSecurity memoryMappedFileSecurity,
- HandleInheritability inheritability, bool leaveOpen)
- */
-
- long fileCapacity = length == 0 ? ushort.MaxValue : length;
-
if (input.memoryMappedFile == null)
{
-#if NETSTANDARD
- input.memoryMappedFile = MemoryMappedFile.CreateFromFile(fc, null, fileCapacity, MemoryMappedFileAccess.ReadWrite, HandleInheritability.Inheritable, false);
-#else
- input.memoryMappedFile = MemoryMappedFile.CreateFromFile(fc, null, fileCapacity, MemoryMappedFileAccess.ReadWrite, null, HandleInheritability.Inheritable, false);
+ input.memoryMappedFile = MemoryMappedFile.CreateFromFile(
+ fileStream: fc,
+ mapName: null,
+ capacity: length,
+ access: MemoryMappedFileAccess.Read,
+#if !NETSTANDARD
+ memoryMappedFileSecurity: null,
#endif
+ inheritability: HandleInheritability.Inheritable,
+ leaveOpen: false);
}
long bufferStart = 0L;
@@ -401,16 +353,21 @@ namespace Lucene.Net.Store
{
int bufSize = (int)((length > (bufferStart + chunkSize)) ? chunkSize : (length - bufferStart));
- // LUCENENET: We get a file access exception if we create a 0 byte file at the end of the range.
- // We can fix this by moving back 1 byte if the bufSize is 0.
+ // LUCENENET: We get an UnauthorizedAccessException if we create a 0 byte file at the end of the range.
+ // See: https://stackoverflow.com/a/5501331
+ // We can fix this by moving back 1 byte on the offset if the bufSize is 0.
int adjust = 0;
if (bufSize == 0 && bufNr == (nrBuffers - 1) && (offset + bufferStart) > 0)
{
adjust = 1;
}
- //buffers[bufNr] = new MemoryMappedFileByteBuffer(input.memoryMappedFile.CreateViewAccessor((offset + bufferStart) - adjust, bufSize, MemoryMappedFileAccess.Read), -1, 0, bufSize, bufSize);
- buffers[bufNr] = new MemoryMappedFileByteBuffer(input.memoryMappedFile.CreateViewAccessor((offset + bufferStart) - adjust, bufSize, MemoryMappedFileAccess.Read), bufSize);
+ buffers[bufNr] = new MemoryMappedFileByteBuffer(
+ input.memoryMappedFile.CreateViewAccessor(
+ offset: (offset + bufferStart) - adjust,
+ size: bufSize,
+ access: MemoryMappedFileAccess.Read),
+ bufSize);
bufferStart += bufSize;
}