Re: [I] Search performance issue with MMapDirectory under load [lucenenet]

["RockNHawk (via GitHub)" <gi...@apache.org>]

RockNHawk commented on issue #1151:
URL: https://github.com/apache/lucenenet/issues/1151#issuecomment-4796900600

   > Well, the good news is nearly all of the classes that are part of the buffers API are internal, including `HeapByteBuffer`. So, if something is sealed it can be unsealed without breaking the public API.
   > 
   > The DotNext code looks interesting. Looks like they are handling `Span<byte>` using this around the public API of `MemoryMappedViewAccessor`.
   > 
   > ![Image](https://github.com/user-attachments/assets/443e465f-2d8b-4245-9400-42983f02d7c0)
   > 
   > Just a thought - it would be a lot simpler if the `ByteBuffer` size were the entire segment since by default `ByteBuffer` is fixed length. Then it could read and/or write locally and `Flush()` would just be to write the entire buffer back to the underlying view/segment.
   
   Yes, we can also obtain a pointer from a MemoryMappedFile and operate on it just like a regular memory pointer.
   
   ```csharp
   
       void Open(string path, string mapName, long size)
       {
           byte* pointer = null;
           if (UseMap)
           {
               MappedFile = MapByFileMode
                   ? MemoryMappedFile.CreateFromFile(path, FileMode.OpenOrCreate, null, size)
                   : MemoryMappedFile.CreateNew($"{VolumeUtility.GetSafePathName(path)}-{mapName}", size);
               Accessor = MappedFile.CreateViewAccessor();
               Accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
           }
           else
           {
               var ptr = Marshal.AllocHGlobal((int)size);
               pointer = (byte*)ptr.ToPointer();
           }
   
           Pointer = pointer;
           MapName = mapName;
           Path = path;
           Size = size;
       }
   
       public void Resize(long newSize)
       {
           if (newSize < Size) throw new ArgumentOutOfRangeException($"New size {newSize} smaller than old size {Size}");
   
           var oldSize = Size;
           var newVersion = Version++;
           var newPath = _initialPath + "_" + newVersion;
           var newMapName = _initialPath + "_" + newVersion;
           byte* newPointer = null;
           if (UseMap)
           {
               //var newMappedFile = MemoryMappedFile.CreateFromFile(newPath, FileMode.Create, null, newSize);
               var newMappedFile = MapByFileMode
                   ? MemoryMappedFile.CreateFromFile(newPath, FileMode.Create, null, newSize)
                   : MemoryMappedFile.CreateNew($"{VolumeUtility.GetSafePathName(newPath)}-{newMapName}", newSize);
               var newAccessor = newMappedFile.CreateViewAccessor();
               newAccessor.SafeMemoryMappedViewHandle.AcquirePointer(ref newPointer);
               try
               {
                   Unsafe.CopyBlockUnaligned(newPointer, Pointer, (uint)oldSize);
                   newAccessor.Flush();
               }
               catch (Exception e)
               {
                   newMappedFile.Dispose();
                   throw;
               }
   
               Dispose();
               System.IO.File.Delete(Path);
               _isDisposed = false;
               MappedFile = newMappedFile;
               Accessor = newAccessor;
           }
           else
           {
               var ptr = Marshal.AllocHGlobal((int)newSize);
               newPointer = (byte*)ptr.ToPointer();
               Buffer.MemoryCopy(Pointer, newPointer, newSize, oldSize);
               // Unsafe.CopyBlockUnaligned(newPointer, Pointer, (uint) oldSize);
               Dispose();
               if (Path != null!) System.IO.File.Delete(Path);
               _isDisposed = false;
           }
   
           Pointer = newPointer;
           Size = newSize;
           Path = newPath;
           MapName = newMapName;
       }
   
   
   
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       public void* ReadPointer(in int offset)
       {
           void* pointer = (byte*)Pointer + offset;
           return pointer;
       }
   
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       public ReadOnlySpan<char> ReadString(in ulong offset, in int length)
       {
           // var fileName = new String((sbyte*)Pointer + offset, 0, length);
           void* pointer = Pointer + offset;
           var fileName = new ReadOnlySpan<char>(pointer, length);
           return fileName;
       }
   
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       public ReadOnlySpan<char> ReadString(in uint offset, in int length)
       {
           void* pointer = Pointer + offset;
           var fileName = new ReadOnlySpan<char>(pointer, length);
           return fileName;
       }
   
   
   
           [MethodImpl(MethodImplOptions.AggressiveInlining)]
           public void Read(in long position, out T value)
           {
               if (position >= Size)
               {
               }
   
               Debug.Assert(position < Size);
               var valueSize = EntrySize;
               byte* ptr = this.Pointer + position;
               value = default(T);
               var valuePtr = Unsafe.AsPointer(ref value);
               Buffer.MemoryCopy(ptr, valuePtr, valueSize, valueSize);
                   // Unsafe.CopyBlockUnaligned(valuePtr, ptr, valueSize);
           }
   
           [MethodImpl(MethodImplOptions.AggressiveInlining)]
           public void Write(in long position, ref T value)
           {
               if (position >= Size)
               {
               }
   
               Debug.Assert(position < Size);
               var valueSize = EntrySize;
               byte* ptr = this.Pointer + position;
               var valuePtr = Unsafe.AsPointer(ref value);
               // Unsafe.CopyBlockUnaligned(ptr, valuePtr, (uint) valueSize);
               Buffer.MemoryCopy(valuePtr, ptr, valueSize, valueSize);
           }
   
           [MethodImpl(MethodImplOptions.AggressiveInlining)]
           public void WriteByIndex(in uint index, ref T value)
           {
               Write((index) * EntrySize, ref value);
           }
   
   
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscribe@lucenenet.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org