You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by pa...@apache.org on 2026/09/01 04:21:08 UTC
(lucenenet) branch master updated: Fix MMapDirectory performance and crashes via raw memory access and drain-reclaiming, #1151, #1013 (#1373)
This is an automated email from the ASF dual-hosted git repository.
paulirwin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new f034c57be Fix MMapDirectory performance and crashes via raw memory access and drain-reclaiming, #1151, #1013 (#1373)
f034c57be is described below
commit f034c57bea09a8ec59579ca82e3daf9b731e0b01
Author: Paul Irwin <pa...@gmail.com>
AuthorDate: Mon Aug 31 22:21:02 2026 -0600
Fix MMapDirectory performance and crashes via raw memory access and drain-reclaiming, #1151, #1013 (#1373)
---
Directory.Build.targets | 5 +-
Lucene.Net.sln.DotSettings | 2 +
.../Lucene.Net.Tests._J-S.csproj | 8 +
src/Lucene.Net.Tests/Store/TestMultiMMap.cs | 2301 +++++++++++++++++++-
.../Store/TestUnsafeChunkIndexInput.cs | 438 ++++
src/Lucene.Net.Tests/Support/TestApiConsistency.cs | 2 +-
src/Lucene.Net.Tests/Support/TestDrainReclaimer.cs | 332 +++
src/Lucene.Net/Store/MMapDirectory.cs | 621 ++++--
src/Lucene.Net/Store/NIOFSDirectory.cs | 4 +
.../Support/Store/UnsafeChunkIndexInput.cs | 566 +++++
src/Lucene.Net/Support/Util/DrainReclaimer.cs | 276 +++
11 files changed, 4332 insertions(+), 223 deletions(-)
diff --git a/Directory.Build.targets b/Directory.Build.targets
index ceb1426fc..5157a5daa 100644
--- a/Directory.Build.targets
+++ b/Directory.Build.targets
@@ -107,14 +107,15 @@
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.1' Or $(TargetFramework.StartsWith('netcoreapp3.')) Or $(TargetFramework.StartsWith('net5.')) Or $(TargetFramework.StartsWith('net6.')) Or $(TargetFramework.StartsWith('net7.')) Or $(TargetFramework.StartsWith('net8.')) Or $(TargetFramework.StartsWith('net9.')) Or $(TargetFramework.StartsWith('net10.')) ">
<DefineConstants>$(DefineConstants);FEATURE_ARRAY_FILL</DefineConstants>
- <DefineConstants>$(DefineConstants);FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_CONDITIONALWEAKTABLE_ADDORUPDATE</DefineConstants>
+ <DefineConstants>$(DefineConstants);FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_ENCODING_GETSTRING_READONLYSPAN</DefineConstants>
+ <DefineConstants>$(DefineConstants);FEATURE_INTERLOCKED_MEMORYBARRIERPROCESSWIDE</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_MEMORYMARSHAL_CREATEREADONLYSPAN</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_NUMBER_PARSE_READONLYSPAN</DefineConstants>
+ <DefineConstants>$(DefineConstants);FEATURE_QUEUE_TRYDEQUEUE_TRYPEEK</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_STREAM_READ_SPAN</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_STRINGBUILDER_APPEND_READONLYSPAN</DefineConstants>
- <DefineConstants>$(DefineConstants);FEATURE_QUEUE_TRYDEQUEUE_TRYPEEK</DefineConstants>
</PropertyGroup>
diff --git a/Lucene.Net.sln.DotSettings b/Lucene.Net.sln.DotSettings
index 970bf1384..6ab5b7a85 100644
--- a/Lucene.Net.sln.DotSettings
+++ b/Lucene.Net.sln.DotSettings
@@ -2,6 +2,8 @@
<s:Boolean x:Key="/Default/UserDictionary/Words/=Coord/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=csharpsquid/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=LUCENENET/@EntryIndexedValue">True</s:Boolean>
+ <s:Boolean x:Key="/Default/UserDictionary/Words/=mmap/@EntryIndexedValue">True</s:Boolean>
+ <s:Boolean x:Key="/Default/UserDictionary/Words/=refcounted/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=stopword/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=stopwords/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=streamutils/@EntryIndexedValue">True</s:Boolean>
diff --git a/src/Lucene.Net.Tests._J-S/Lucene.Net.Tests._J-S.csproj b/src/Lucene.Net.Tests._J-S/Lucene.Net.Tests._J-S.csproj
index 9527dc513..2b6aee47e 100644
--- a/src/Lucene.Net.Tests._J-S/Lucene.Net.Tests._J-S.csproj
+++ b/src/Lucene.Net.Tests._J-S/Lucene.Net.Tests._J-S.csproj
@@ -37,6 +37,14 @@
<Compile Include="..\Lucene.Net.Tests\Support\**\*.cs" LinkBase="Support" Exclude="..\Lucene.Net.Tests\Support\ExceptionHandling\**\*" />
<Compile Include="..\Lucene.Net.Tests\Util\TestNumericUtils.cs" Link="Util\TestNumericUtils.cs" />
<EmbeddedResource Include="..\Lucene.Net.Tests\Store\LUCENENET521.zip" Link="Store\LUCENENET521.zip" />
+ <!--
+ 3.x CFS index fixtures, re-linked into the Store namespace so
+ TestMultiMMap can exercise the MMapDirectory IndexInputSlicer
+ OpenFullSlice path used by CompoundFileDirectory when reading
+ CFS files from 3.x indexes.
+ -->
+ <EmbeddedResource Include="..\Lucene.Net.Tests\Index\index.30.cfs.zip" Link="Store\index.30.cfs.zip" />
+ <EmbeddedResource Include="..\Lucene.Net.Tests\Index\index.31.cfs.zip" Link="Store\index.31.cfs.zip" />
</ItemGroup>
<ItemGroup>
diff --git a/src/Lucene.Net.Tests/Store/TestMultiMMap.cs b/src/Lucene.Net.Tests/Store/TestMultiMMap.cs
index 8a2fbc7e6..bccd7887d 100644
--- a/src/Lucene.Net.Tests/Store/TestMultiMMap.cs
+++ b/src/Lucene.Net.Tests/Store/TestMultiMMap.cs
@@ -1,12 +1,17 @@
+using J2N;
using Lucene.Net.Attributes;
using Lucene.Net.Documents;
using Lucene.Net.Index.Extensions;
using NUnit.Framework;
using System;
+using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
+using System.Threading.Tasks;
using Assert = Lucene.Net.TestFramework.Assert;
namespace Lucene.Net.Store
@@ -28,22 +33,22 @@ namespace Lucene.Net.Store
* limitations under the License.
*/
- using BytesRef = Lucene.Net.Util.BytesRef;
- using Document = Documents.Document;
+ using BytesRef = Util.BytesRef;
+ using Document = Document;
using Field = Field;
- using IndexInputSlicer = Lucene.Net.Store.Directory.IndexInputSlicer;
- using IndexReader = Lucene.Net.Index.IndexReader;
- using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
- using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
- using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
- using TestUtil = Lucene.Net.Util.TestUtil;
+ using IndexInputSlicer = Directory.IndexInputSlicer;
+ using IndexReader = Index.IndexReader;
+ using LuceneTestCase = Util.LuceneTestCase;
+ using MockAnalyzer = Analysis.MockAnalyzer;
+ using RandomIndexWriter = Index.RandomIndexWriter;
+ using TestUtil = Util.TestUtil;
/// <summary>
/// Tests MMapDirectory's MultiMMapIndexInput
- /// <p>
- /// Because Java's ByteBuffer uses an int to address the
- /// values, it's necessary to access a file >
- /// Integer.MAX_VALUE in size using multiple byte buffers.
+ /// <para/>
+ /// Because .NET's <see cref="Span{T}"/> and <see cref="ReadOnlySpan{T}"/> use an int to address the
+ /// values, and because we use a similar chunking approach to Lucene, it's necessary to access a file >
+ /// <see cref="Int32.MaxValue"/> in size using multiple byte buffers.
/// </summary>
[TestFixture]
public class TestMultiMMap : LuceneTestCase
@@ -225,6 +230,79 @@ public virtual void TestCloneSliceClose()
mmapDir.Dispose();
}
+ // LUCENENET specific: exercises the shared MemoryMappedFile refactor
+ // where OpenInput, CreateSlicer, its slices, and clones all piggyback
+ // on a single MemoryMappedFile per file (per directory instance).
+ // Verifies that (a) concurrent IndexInputs all see correct bytes,
+ // (b) disposing in arbitrary order keeps siblings functional, and
+ // (c) once the last referrer is disposed the OS handle is released
+ // (on Windows a still-open mapping would prevent the file delete).
+ [Test, LuceneNetSpecific]
+ public virtual void TestSharedMappingLifecycle()
+ {
+ var tempDir = CreateTempDir("testSharedMappingLifecycle");
+ MMapDirectory mmapDir = new MMapDirectory(tempDir);
+ const string name = "bytes";
+ using (IndexOutput io = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ // 4 ints at offsets 0, 4, 8, 12 — each slice reads a known value.
+ io.WriteInt32(10);
+ io.WriteInt32(20);
+ io.WriteInt32(30);
+ io.WriteInt32(40);
+ }
+
+ // Open several IndexInputs for the same file through both
+ // OpenInput and CreateSlicer. All should share one mapping.
+ IndexInput root = mmapDir.OpenInput(name, IOContext.DEFAULT);
+ IndexInput rootClone = (IndexInput)root.Clone();
+
+ IndexInputSlicer slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
+ IndexInput sliceA = slicer.OpenSlice("a", 0, 4);
+ IndexInput sliceB = slicer.OpenSlice("b", 8, 4);
+ IndexInput sliceAClone = (IndexInput)sliceA.Clone();
+
+ // Reads across all instances must be independent and correct.
+ Assert.AreEqual(10, root.ReadInt32());
+ Assert.AreEqual(10, rootClone.ReadInt32());
+ Assert.AreEqual(10, sliceA.ReadInt32());
+ Assert.AreEqual(30, sliceB.ReadInt32());
+ Assert.AreEqual(10, sliceAClone.ReadInt32());
+
+ // Dispose a clone first; the root and siblings must keep working.
+ rootClone.Dispose();
+ root.Seek(4);
+ Assert.AreEqual(20, root.ReadInt32());
+ sliceB.Seek(0);
+ Assert.AreEqual(30, sliceB.ReadInt32());
+
+ // Dispose a slice; its siblings from the same slicer must keep working.
+ sliceAClone.Dispose();
+ sliceA.Seek(0);
+ Assert.AreEqual(10, sliceA.ReadInt32());
+
+ // Dispose the remaining slice-side instances. The root IndexInput
+ // owns its own mapping, so it must stay alive and readable.
+ sliceA.Dispose();
+ sliceB.Dispose();
+ slicer.Dispose();
+
+ root.Seek(12);
+ Assert.AreEqual(40, root.ReadInt32());
+
+ // Disposing the root tears down its MemoryMappedFile and backing
+ // FileStream.
+ root.Dispose();
+
+ // If any OS file handle is still open, this delete will fail on
+ // Windows. On Unix it silently unlinks but the test still proves
+ // the read-phase invariants above.
+ mmapDir.DeleteFile(name);
+ Assert.IsFalse(File.Exists(Path.Combine(tempDir.FullName, name)));
+
+ mmapDir.Dispose();
+ }
+
[Test]
public virtual void TestSeekZero()
{
@@ -419,16 +497,34 @@ private void AssertChunking(Random random, int chunkSize)
// LUCENENET: Regression test for GitHub #1090. A background thread
// extends a file on disk while the foreground thread repeatedly
- // opens it with MMapDirectory.OpenInput. Before the fix, the
- // file's length could grow between the caller capturing fc.Length
- // and MemoryMappedFile.CreateFromFile performing its internal
- // stat, causing ArgumentOutOfRangeException (paramName="capacity")
- // with the message "The capacity may not be smaller than the
- // file size."
- // NonParallelizable: the retry-path assertion reads static counters on
- // MMapDirectory, so any other test exercising MMapDirectory in parallel
- // could skew the observed retry count.
- [Test, LuceneNetSpecific, Slow, NonParallelizable]
+ // opens it with MMapDirectory.OpenInput. The original failure
+ // mode was ArgumentOutOfRangeException(paramName="capacity")
+ // from MemoryMappedFile.CreateFromFile, because the on-disk file
+ // size could exceed our caller-computed capacity by the time the
+ // framework did its internal stat. .NET Framework's
+ // CreateFromFile reads fileStream.Length multiple times
+ // non-atomically (referencesource MemoryMappedFile.cs L192-L243);
+ // modern .NET snapshots it into a single local
+ // (dotnet/runtime MemoryMappedFile.cs L237-L268). Even when we
+ // pass capacity: 0 the .NET Framework path still races because
+ // the length is re-read for both the defaulting step and the
+ // capacity-vs-size guard. SharedMapping.Create handles the
+ // residual race with a retry loop. This test asserts that
+ // OpenInput continues to succeed under concurrent file
+ // extension.
+ //
+ // Test design notes:
+ // - The writer extends only (never truncates). Truncating a
+ // user-mapped file on Windows fails with ERROR_USER_MAPPED_FILE
+ // and is unrelated to what we're verifying here.
+ // - The reader runs a bounded number of iterations rather than a
+ // wall-clock loop. Sustained mmap churn (thousands of
+ // map/unmap pairs per second) can transiently exhaust Windows
+ // kernel resources (ERROR_NO_SYSTEM_RESOURCES,
+ // ERROR_ACCESS_DENIED on view creation), which is also
+ // unrelated to the capacity race. A few hundred iterations
+ // are plenty to repeatedly hit the race window.
+ [Test, LuceneNetSpecific, Nightly]
public void TestOpenInputConcurrentFileExtension_Issue1090()
{
var dir = CreateTempDir("testOpenInputConcurrentFileExtension");
@@ -440,8 +536,8 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
using var mmapDir = new MMapDirectory(dir);
- const long maxFileSize = 1L * 1024 * 1024; // 1 MiB cap
- var stop = new ManualResetEventSlim(false);
+ const long maxFileSize = 64L * 1024 * 1024; // 64 MiB safety cap
+ using var stop = new ManualResetEventSlim(false);
Exception writerError = null;
var writer = new Thread(() =>
@@ -449,19 +545,19 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
var chunk = new byte[64];
try
{
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
while (!stop.IsSet)
{
using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
fs.Seek(0, SeekOrigin.End);
- if (fs.Length < maxFileSize)
- {
- fs.Write(chunk, 0, chunk.Length);
- }
- else
+ if (fs.Length >= maxFileSize)
{
- // Keep the file bounded: truncate back and grow again.
- fs.SetLength(64);
+ // Stop extending if we somehow reach the cap. The
+ // reader's bounded iteration count guarantees this
+ // is far above what we'll hit in a normal run.
+ break;
}
+ fs.Write(chunk, 0, chunk.Length);
}
}
catch (Exception e)
@@ -472,80 +568,2135 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
{ IsBackground = true, Name = "mmap-issue1090-extender" };
writer.Start();
- // Snapshot counters so this test's assertion is not affected by
- // any earlier test's activity on MMapDirectory.
- long baselineRetries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount);
-
try
{
- var sw = Stopwatch.StartNew();
- int iterations = 0;
- // Keep stretching the window until either the race fires or we
- // hit a hard deadline. On most machines this takes < 1 second.
- const int maxSeconds = 15;
- while (sw.Elapsed < TimeSpan.FromSeconds(maxSeconds))
+ // Bounded iteration count keeps mmap churn well below the
+ // Windows kernel-resource threshold while still exercising
+ // the capacity race many times over.
+ const int iterations = 500;
+ for (int i = 0; i < iterations; i++)
{
using (var _ = mmapDir.OpenInput(name, NewIOContext(Random)))
{
// Just open and dispose; the race occurs during construction.
}
- iterations++;
+ }
+ }
+ finally
+ {
+ stop.Set();
+ writer.Join();
+ }
+
+ if (writerError != null)
+ {
+ throw new Exception("Writer thread failed", writerError);
+ }
+ }
+
+ // Regression test for issue #1013: sporadic AccessViolationException
+ // during concurrent search with SearcherManager on MMapDirectory.
+ //
+ // Strategy: spin many reader threads cloning + reading a shared
+ // IndexInput while another thread disposes it mid-flight. The
+ // invariant under test: concurrent Clone/read against a Dispose
+ // must only ever surface AlreadyClosed-style exceptions — never an
+ // AVE (which crashes the test host), never an NRE, never an IOE
+ // from a half-torn-down mapping.
+ //
+ // Under the chunked, reclaimer-backed design, clones observe the closed
+ // mapping (a chunk crossing after close throws) and throw AlreadyClosed
+ // promptly after the root is disposed. A successful pass here is
+ // therefore a *positive* result — not Inconclusive — because the
+ // expected behavior is that the invariant holds throughout.
+ // [Nightly]: wall-clock stress loop (up to ~30s). Kept out of the
+ // default run so CI isn't lengthened, but exercised in nightly runs
+ // where catching regressions in the #1013 race path is worth the time.
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestConcurrentCloneReadVsDispose_Issue1013()
+ {
+ var dirPath = CreateTempDir("testIssue1013");
+ using var mmapDir = new MMapDirectory(dirPath);
+
+ const string name = "bytes";
+ const int fileSize = 1 << 20; // 1 MiB
+ var random = Random;
+
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int written = 0; written < fileSize; written += buf.Length)
+ {
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+ }
+
+ const int readerThreads = 8;
+ const int maxSeconds = 30;
+
+ var sw = Stopwatch.StartNew();
+ int iteration = 0;
+ int raceObserved = 0;
+ var unexpectedExceptions = new ConcurrentBag<Exception>();
- if (Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) > baselineRetries)
+ while (sw.Elapsed < TimeSpan.FromSeconds(maxSeconds) && raceObserved == 0)
+ {
+ iteration++;
+
+ var primary = mmapDir.OpenInput(name, NewIOContext(random));
+ using var start = new ManualResetEventSlim(false);
+ var threads = new Thread[readerThreads];
+ long totalReads = 0;
+
+ for (int i = 0; i < readerThreads; i++)
+ {
+ threads[i] = new Thread(() =>
{
- break; // race reproduced and handled by the retry loop
- }
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ try
+ {
+ while (true)
+ {
+ IndexInput clone;
+ try
+ {
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ clone = (IndexInput)primary.Clone();
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ return;
+ }
+
+ try
+ {
+ for (int p = 0; p < fileSize; p++)
+ {
+ clone.ReadByte();
+ Interlocked.Increment(ref totalReads);
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ return;
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ unexpectedExceptions.Add(e);
+ Interlocked.Exchange(ref raceObserved, 1);
+ }
+ })
+ { IsBackground = true, Name = $"issue1013-reader-{i}" };
+ threads[i].Start();
}
- long retries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) - baselineRetries;
- int maxAttempts = Volatile.Read(ref MMapDirectory.s_maxCapacityAttemptsObserved);
+ start.Set();
- // Surface what was observed for diagnostics when run with -v normal.
- TestContext.Progress.WriteLine(
- $"TestOpenInputConcurrentFileExtension: iterations={iterations}, retries={retries}, maxAttemptsObserved={maxAttempts}");
+ Thread.Sleep(random.Next(1, 5));
+ primary.Dispose();
+
+ // Join every reader. The reclaimer blocks the unmap until
+ // in-flight reads drain, and clones that cross into a closed
+ // chunk observe it and exit via the expected-AlreadyClosed
+ // catch. So Join timing out would itself be a defect - either
+ // the reclaimer is leaking or a reader is stuck in a broken
+ // state. Record that and fail rather than silently abandoning
+ // the thread.
+ foreach (var t in threads)
+ {
+ if (!t.Join(TimeSpan.FromSeconds(10)))
+ {
+ unexpectedExceptions.Add(new TimeoutException(
+ $"Reader thread {t.Name} did not exit within 10s after primary.Dispose(); " +
+ "expected AlreadyClosed to propagate out of the read path."));
+ Interlocked.Exchange(ref raceObserved, 1);
+ // Continue joining the rest so we don't leak live
+ // threads holding IndexInput clones into later
+ // iterations or subsequent tests.
+ }
+ }
- // The real check: the race must have fired and our retry loop
- // must have swallowed it. Without the fix, the exception
- // escapes OpenInput and the test fails with ArgumentOutOfRangeException
- // (as seen in #1090). If the race never fires during this run
- // (timing-dependent), mark the test inconclusive rather than
- // silently passing — we haven't actually exercised the fix.
- if (retries == 0)
+ if (iteration % 50 == 0)
{
- NUnit.Framework.Assert.Inconclusive(
- $"The concurrent-extension race was not reproduced within {maxSeconds}s " +
- $"({iterations} OpenInput iterations). The fix was therefore not exercised on this run.");
+ TestContext.Progress.WriteLine(
+ $"issue1013 repro: iteration={iteration}, elapsed={sw.Elapsed.TotalSeconds:0.0}s, reads={totalReads}");
}
}
- finally
+
+ if (raceObserved != 0)
+ {
+ var example = unexpectedExceptions.FirstOrDefault();
+ Assert.Fail(
+ $"Issue #1013 invariant violated on iteration {iteration}: " +
+ $"concurrent clone/read vs Dispose produced an unexpected exception type. " +
+ $"Example: {example?.GetType().FullName}: {example?.Message}\n{example}");
+ }
+
+ Assert.Pass(
+ $"Issue #1013 invariant held across {iteration} iterations in " +
+ $"{sw.Elapsed.TotalSeconds:0.0}s — no AVE / NRE / unexpected exception " +
+ "under concurrent clone/read vs Dispose.");
+ }
+
+ // LUCENENET-specific (#1013): the disposing thread is the SAME thread that
+ // owns and is reading the primary, while OTHER threads concurrently read
+ // clones that share the primary's mapping. This pins the invariant that a
+ // same-thread Dispose is NOT the Java unmap-hack: disposing the owning input
+ // closes the shared mapping (requesting its unmap), but the DrainReclaimer
+ // blocks the actual unmap until every in-flight reader drains, so concurrent
+ // readers on sibling clones are never left dereferencing a freed view.
+ // Expected outcomes for the sibling readers: valid bytes, or AlreadyClosed
+ // once they cross into a closed chunk. Never an AVE, NRE, or
+ // torn-down-mapping IOException.
+ // [Nightly]: wall-clock stress loop (~15s).
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestSameThreadOwnerDisposeWhileSiblingClonesRead_NoAVE()
+ {
+ var dirPath = CreateTempDir("testSameThreadDisposeVsSiblingReads");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ const int fileSize = 1 << 20; // 1 MiB, spans multiple chunks
+ var random = Random;
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int w = 0; w < fileSize; w += buf.Length)
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+
+ const int siblingReaders = 6;
+ var unexpected = new ConcurrentBag<Exception>();
+ int iterations = 0;
+ var sw = Stopwatch.StartNew();
+
+ while (sw.Elapsed < TimeSpan.FromSeconds(15) && unexpected.IsEmpty)
{
+ iterations++;
+
+ // The primary is opened, read, AND disposed all on THIS thread.
+ var primary = mmapDir.OpenInput(name, NewIOContext(random));
+
+ using var start = new ManualResetEventSlim(false);
+ using var stop = new ManualResetEventSlim(false);
+ var readers = new Thread[siblingReaders];
+ for (int i = 0; i < readers.Length; i++)
+ {
+ readers[i] = new Thread(() =>
+ {
+ // Each sibling reads its OWN clone, which shares primary's
+ // mapping. Clone before the barrier; if the primary is
+ // already disposed (later iterations race), Clone throws
+ // AlreadyClosed, which is an acceptable outcome.
+ IndexInput clone;
+ try
+ {
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ clone = (IndexInput)primary.Clone();
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ return;
+ }
+
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ try
+ {
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ while (!stop.IsSet)
+ {
+ clone.Seek(0);
+ for (int p = 0; p < fileSize; p++)
+ clone.ReadByte();
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // Expected once the owner disposes and we cross into a
+ // closed chunk.
+ }
+ catch (Exception e)
+ {
+ unexpected.Add(e);
+ }
+ })
+ { IsBackground = true, Name = $"sibling-reader-{i}" };
+ readers[i].Start();
+ }
+
+ start.Set();
+
+ // The owning thread reads the primary itself for a beat, then
+ // disposes it SAME-THREAD while the siblings are mid-read.
+ try
+ {
+ primary.Seek(0);
+ for (int p = 0; p < fileSize && p < 64 * 1024; p++)
+ primary.ReadByte();
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // Not expected here (we haven't disposed yet), but harmless.
+ }
+
+ primary.Dispose(); // same-thread close of the owning input
stop.Set();
- writer.Join();
+
+ foreach (var t in readers)
+ {
+ if (!t.Join(TimeSpan.FromSeconds(10)))
+ {
+ unexpected.Add(new TimeoutException(
+ $"Sibling reader {t.Name} did not exit within 10s after the " +
+ "owner's same-thread Dispose; expected AlreadyClosed to propagate."));
+ }
+ }
}
- if (writerError != null)
+ if (!unexpected.IsEmpty)
{
- throw new Exception("Writer thread failed", writerError);
+ var ex = unexpected.First();
+ Assert.Fail(
+ $"Same-thread owner Dispose vs concurrent sibling-clone reads produced an " +
+ $"unexpected exception after {iterations} iterations: " +
+ $"{ex.GetType().FullName}: {ex.Message}\n{ex}");
+ }
+
+ Assert.Pass(
+ $"Same-thread owner Dispose did not AVE concurrent sibling readers across " +
+ $"{iterations} iterations in {sw.Elapsed.TotalSeconds:0.0}s.");
+ }
+
+ // LUCENENET-specific: race-condition coverage for the chunked,
+ // reclaimer-backed MMapIndexInput. These tests complement the
+ // single-threaded TestCloneClose / TestCloneSliceSafety /
+ // TestCloneSliceClose tests by exercising concurrent Dispose vs.
+ // read, Dispose vs. Clone, and slicer-cascade scenarios.
+
+ // Concurrent Clone during Dispose: the root is disposed while many
+ // threads repeatedly call Clone() + ReadByte(). Invariants:
+ // - No AVE / NRE / memory corruption.
+ // - Once primary.Dispose has returned and the cloner thread has
+ // observed that, subsequent reads on its current clone throw
+ // AlreadyClosed.
+ // - After join, calling Clone() + read on the disposed primary
+ // from the main thread throws AlreadyClosed — pinning that a
+ // disposed root cannot silently hand out a working clone.
+ // [Nightly]: wall-clock stress loop (~15s). See Issue1013 rationale.
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestConcurrentCloneVsDispose_RaceScenario()
+ {
+ var dirPath = CreateTempDir("testCloneVsDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ const int fileSize = 64 * 1024;
+ var random = Random;
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int w = 0; w < fileSize; w += buf.Length)
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+
+ var unexpected = new ConcurrentBag<Exception>();
+ int iterations = 0;
+ var sw = Stopwatch.StartNew();
+ while (sw.Elapsed < TimeSpan.FromSeconds(15))
+ {
+ iterations++;
+ var primary = mmapDir.OpenInput(name, NewIOContext(random));
+ using var start = new ManualResetEventSlim(false);
+ var cloners = new Thread[6];
+ for (int i = 0; i < cloners.Length; i++)
+ {
+ cloners[i] = new Thread(() =>
+ {
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ try
+ {
+ while (true)
+ {
+ IndexInput c;
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ try { c = (IndexInput)primary.Clone(); }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+ // Touch a byte on the clone — but don't read past dispose to keep the test focused on Clone itself.
+ try { c.ReadByte(); }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+ }
+ }
+ catch (Exception e) { unexpected.Add(e); }
+ }) { IsBackground = true };
+ cloners[i].Start();
+ }
+ start.Set();
+ Thread.Sleep(random.Next(0, 3));
+ primary.Dispose();
+ foreach (var t in cloners)
+ {
+ if (!t.Join(TimeSpan.FromSeconds(5)))
+ {
+ unexpected.Add(new TimeoutException(
+ "Cloner thread did not exit within 5s after primary.Dispose()."));
+ }
+ }
+
+ // Positive contract check: after Dispose, Clone() on the
+ // disposed root either throws AlreadyClosed or produces a
+ // clone whose first read throws AlreadyClosed. The failure
+ // mode we want to catch is a clone that silently hands back
+ // bytes from a released mapping.
+ try
+ {
+ var postDisposeClone = (IndexInput)primary.Clone();
+ try
+ {
+ postDisposeClone.ReadByte();
+ unexpected.Add(new InvalidOperationException(
+ "Clone() + ReadByte() on disposed primary returned without throwing AlreadyClosed."));
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // also acceptable: Clone() itself refused
+ }
+ }
+
+ if (!unexpected.IsEmpty)
+ {
+ var ex = unexpected.First();
+ Assert.Fail($"Concurrent Clone-vs-Dispose produced unexpected exception after {iterations} iterations: {ex.GetType().FullName}: {ex.Message}\n{ex}");
+ }
+ }
+
+ // Concurrent read of the SAME instance during Dispose of that
+ // instance. Drain-barrier must prevent the disposer from releasing
+ // the pointer while a reader is mid-CopyBlockUnaligned.
+ // [Nightly]: wall-clock stress loop (~15s). See Issue1013 rationale.
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestConcurrentReadVsSelfDispose_RaceScenario()
+ {
+ var dirPath = CreateTempDir("testReadVsSelfDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ const int fileSize = 1 << 18; // 256 KiB — enough for several buffer refills
+ var random = Random;
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int w = 0; w < fileSize; w += buf.Length)
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+
+ var unexpected = new ConcurrentBag<Exception>();
+ int iterations = 0;
+ var sw = Stopwatch.StartNew();
+ while (sw.Elapsed < TimeSpan.FromSeconds(15) && unexpected.IsEmpty)
+ {
+ iterations++;
+ var input = mmapDir.OpenInput(name, NewIOContext(random));
+ using var start = new ManualResetEventSlim(false);
+ var readers = new Thread[4];
+ for (int i = 0; i < readers.Length; i++)
+ {
+ readers[i] = new Thread(() =>
+ {
+ // Each reader uses its OWN clone — contract says
+ // IndexInput isn't thread-safe. We want to stress
+ // the reclaimer on the shared mapping, not a single
+ // IndexInput.
+ IndexInput clone;
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ try { clone = (IndexInput)input.Clone(); }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ try
+ {
+ while (true)
+ {
+ clone.Seek(0);
+ for (int p = 0; p < fileSize; p++)
+ {
+ clone.ReadByte();
+ }
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+ catch (Exception e) { unexpected.Add(e); }
+ }) { IsBackground = true };
+ readers[i].Start();
+ }
+ start.Set();
+ Thread.Sleep(random.Next(1, 5));
+ input.Dispose();
+ foreach (var t in readers) t.Join(TimeSpan.FromSeconds(5));
+ }
+
+ if (!unexpected.IsEmpty)
+ {
+ var ex = unexpected.First();
+ Assert.Fail($"Concurrent read-vs-self-dispose produced unexpected exception after {iterations} iterations: {ex.GetType().FullName}: {ex.Message}\n{ex}");
+ }
+ }
+
+ // Slicer Dispose while slices are being read concurrently. The
+ // slicer cascades Dispose to all issued slices; every in-flight
+ // reader must either finish its current CopyBlockUnaligned or
+ // observe the closed state cleanly (no AVE).
+ // [Nightly]: wall-clock stress loop (~15s). See Issue1013 rationale.
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestConcurrentSliceReadVsSlicerDispose_RaceScenario()
+ {
+ var dirPath = CreateTempDir("testSliceVsSlicerDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ const int fileSize = 1 << 18;
+ var random = Random;
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int w = 0; w < fileSize; w += buf.Length)
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+
+ var unexpected = new ConcurrentBag<Exception>();
+ int iterations = 0;
+ var sw = Stopwatch.StartNew();
+ while (sw.Elapsed < TimeSpan.FromSeconds(15) && unexpected.IsEmpty)
+ {
+ iterations++;
+ var slicer = mmapDir.CreateSlicer(name, NewIOContext(random));
+ using var start = new ManualResetEventSlim(false);
+ var readers = new Thread[4];
+ for (int i = 0; i < readers.Length; i++)
+ {
+ int sliceIndex = i;
+ readers[i] = new Thread(() =>
+ {
+ IndexInput slice;
+ try
+ {
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ slice = slicer.OpenSlice("slice" + sliceIndex, 0, fileSize);
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ try
+ {
+ while (true)
+ {
+ slice.Seek(0);
+ for (int p = 0; p < fileSize; p++)
+ slice.ReadByte();
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException()) { return; }
+ catch (Exception e) { unexpected.Add(e); }
+ }) { IsBackground = true };
+ readers[i].Start();
+ }
+ start.Set();
+ Thread.Sleep(random.Next(1, 5));
+ slicer.Dispose();
+ foreach (var t in readers) t.Join(TimeSpan.FromSeconds(5));
+ }
+
+ if (!unexpected.IsEmpty)
+ {
+ var ex = unexpected.First();
+ Assert.Fail($"Concurrent slice-read-vs-slicer-dispose produced unexpected exception after {iterations} iterations: {ex.GetType().FullName}: {ex.Message}\n{ex}");
}
}
+ // Empty (zero-byte) file edge case: OpenInput/Clone/Dispose must not
+ // AVE or throw. Zero-length files take a different MapAndAcquire
+ // path (no MMF, no AcquirePointer), so ReadInternal hits the
+ // basePtr==null branch and must handle a zero-byte read cleanly
+ // while rejecting a non-zero read with EOF.
[Test, LuceneNetSpecific]
- public void TestDisposeIndexInput()
+ public void TestZeroLengthFile_ReadsAndCloneAndDispose()
{
- string name = "foobar";
- var dir = CreateTempDir("testDisposeIndexInput");
- string fileName = Path.Combine(dir.FullName, name);
+ var dirPath = CreateTempDir("testZeroLengthFile");
+ using var mmapDir = new MMapDirectory(dirPath);
+ using (var io = mmapDir.CreateOutput("empty", NewIOContext(Random))) { }
- // Create a zero byte file, and close it immediately
- File.WriteAllText(fileName, string.Empty, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false) /* No BOM */);
+ using var input = mmapDir.OpenInput("empty", NewIOContext(Random));
+ Assert.AreEqual(0L, input.Length);
+ using var clone = (IndexInput)input.Clone();
+ Assert.AreEqual(0L, clone.Length);
+ Assert.AreEqual(0L, input.Position);
+ Assert.AreEqual(0L, clone.Position);
- MMapDirectory mmapDir = new MMapDirectory(dir);
- using (var _ = mmapDir.OpenInput(name, NewIOContext(Random)))
+ // Exercise the ReadInternal basePtr==null, destination.Length==0
+ // branch. This should succeed silently.
+ input.ReadBytes(Array.Empty<byte>(), 0, 0);
+ clone.ReadBytes(Array.Empty<byte>(), 0, 0);
+
+ // Any non-zero read must throw EOF (we can't satisfy it from 0
+ // bytes of mapped data).
+ try
{
- } // Dispose
+ input.ReadByte();
+ Assert.Fail("ReadByte on zero-length file must throw EOF");
+ }
+ catch (EndOfStreamException)
+ {
+ // expected
+ }
+ }
- // Now it should be possible to delete the file. This is the condition we are testing for.
- File.Delete(fileName);
+ // Read-after-Dispose on a single instance (not concurrent): the new
+ // class must throw AlreadyClosedException rather than reading stale
+ // bytes from the unmapped region. This is the single-threaded
+ // correctness analogue of #1013.
+ [Test, LuceneNetSpecific]
+ public void TestReadAfterDispose_ThrowsAlreadyClosed()
+ {
+ var dirPath = CreateTempDir("testReadAfterDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ using (var io = mmapDir.CreateOutput("bytes", NewIOContext(Random)))
+ {
+ io.WriteInt32(42);
+ }
+
+ var input = mmapDir.OpenInput("bytes", NewIOContext(Random));
+ input.Dispose();
+ try
+ {
+ input.ReadInt32();
+ Assert.Fail("Must throw AlreadyClosedException after Dispose");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // pass
+ }
+ }
+
+ // Clone-after-Dispose: pins the current contract that Clone() on a
+ // disposed root does NOT throw (MemberwiseClone is just a managed
+ // object copy), but the cloned IndexInput observes the closed
+ // mapping on its first read and throws AlreadyClosed. Likewise,
+ // disposing a clone does not close the shared mapping, so the root
+ // + sibling clones stay alive.
+ [Test, LuceneNetSpecific]
+ public void TestCloneAfterDispose_ReadsThrowAlreadyClosed()
+ {
+ var dirPath = CreateTempDir("testCloneAfterDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ using (var io = mmapDir.CreateOutput("bytes", NewIOContext(Random)))
+ {
+ io.WriteInt32(1);
+ io.WriteInt32(2);
+ }
+
+ var root = mmapDir.OpenInput("bytes", NewIOContext(Random));
+
+ // Clone taken BEFORE the root is disposed: must fail on read
+ // after the root is disposed, because the shared mapping is closed.
+ var preDisposeClone = (IndexInput)root.Clone();
+
+ root.Dispose();
+
+ try
+ {
+ preDisposeClone.ReadInt32();
+ Assert.Fail("Pre-dispose clone must throw AlreadyClosed after root Dispose");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+
+ // Clone taken AFTER the root is disposed: Clone() itself must
+ // throw AlreadyClosed (we check the parent's instanceClosed
+ // flag at the top of Clone, matching upstream Java's behavior
+ // of failing fast rather than deferring to the first read).
+ try
+ {
+ root.Clone();
+ Assert.Fail("Clone of disposed root must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ }
+
+ // Sibling isolation: disposing one clone does NOT close the shared
+ // View, so the root and other clones continue to read. Counterpart
+ // to TestCloneAfterDispose — covers the case where a clone is
+ // disposed first, rather than the root.
+ [Test, LuceneNetSpecific]
+ public void TestDisposingCloneDoesNotAffectRootOrSiblings()
+ {
+ var dirPath = CreateTempDir("testCloneSiblingIsolation");
+ using var mmapDir = new MMapDirectory(dirPath);
+ using (var io = mmapDir.CreateOutput("bytes", NewIOContext(Random)))
+ {
+ io.WriteInt32(1);
+ io.WriteInt32(2);
+ }
+
+ using var root = mmapDir.OpenInput("bytes", NewIOContext(Random));
+ var cloneA = (IndexInput)root.Clone();
+ var cloneB = (IndexInput)root.Clone();
+
+ cloneA.Dispose();
+
+ // Root and sibling clone must still work.
+ Assert.AreEqual(1, root.ReadInt32());
+ Assert.AreEqual(1, cloneB.ReadInt32());
+
+ // Disposed clone must throw on read.
+ try
+ {
+ cloneA.ReadInt32();
+ Assert.Fail("Disposed clone must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+
+ cloneB.Dispose();
+ }
+
+ // Multiple slices over the same file must each see their own region
+ // of data, not stale bytes from a neighboring slice. Each OpenSlice
+ // in the new design opens its own MemoryMappedFile+view over the
+ // given (offset, length) window, so bounds are enforced per-slice
+ // and reads are independent.
+ [Test, LuceneNetSpecific]
+ public void TestMultipleSlicesReadDistinctData()
+ {
+ var dirPath = CreateTempDir("testMultipleSlicesDistinct");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ const int regionSize = 4096;
+ const int regions = 4;
+
+ // Write four contiguous 4KiB regions, each filled with a distinct
+ // byte pattern (0x11, 0x22, 0x33, 0x44). Later we open a slice
+ // over each region and check that reads return the right pattern.
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ var buf = new byte[regionSize];
+ for (int r = 0; r < regions; r++)
+ {
+ byte fill = (byte)(0x11 * (r + 1));
+ for (int i = 0; i < buf.Length; i++) buf[i] = fill;
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+ }
+
+ using var slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
+
+ var slices = new IndexInput[regions];
+ try
+ {
+ for (int r = 0; r < regions; r++)
+ {
+ slices[r] = slicer.OpenSlice("slice" + r, r * regionSize, regionSize);
+ Assert.AreEqual(regionSize, slices[r].Length, $"slice {r} length");
+ }
+
+ // Each slice sees only its own pattern.
+ for (int r = 0; r < regions; r++)
+ {
+ byte expected = (byte)(0x11 * (r + 1));
+ for (int i = 0; i < regionSize; i++)
+ {
+ byte b = slices[r].ReadByte();
+ if (b != expected)
+ {
+ Assert.Fail(
+ $"slice {r} at offset {i}: expected 0x{expected:X2}, got 0x{b:X2}");
+ }
+ }
+
+ // Reading past the slice's length must fail with EOF.
+ try
+ {
+ slices[r].ReadByte();
+ Assert.Fail($"slice {r}: read past end of slice must throw EOF");
+ }
+ catch (EndOfStreamException)
+ {
+ // expected
+ }
+ }
+
+ // Concurrent reads across sibling slices: each worker scans
+ // its own slice fully and asserts the pattern. If the slices
+ // were accidentally aliased to the same underlying view,
+ // racing Seek() calls would cross-contaminate.
+ var errors = new ConcurrentBag<string>();
+ var tasks = new Task[regions];
+ for (int r = 0; r < regions; r++)
+ {
+ int idx = r;
+ byte expected = (byte)(0x11 * (idx + 1));
+ var clone = (IndexInput)slices[idx].Clone();
+ tasks[idx] = Task.Run(() =>
+ {
+ for (int pass = 0; pass < 50; pass++)
+ {
+ clone.Seek(0);
+ for (int i = 0; i < regionSize; i++)
+ {
+ byte b = clone.ReadByte();
+ if (b != expected)
+ {
+ errors.Add($"slice {idx} pass {pass} offset {i}: expected 0x{expected:X2}, got 0x{b:X2}");
+ return;
+ }
+ }
+ }
+ });
+ }
+ Task.WaitAll(tasks);
+ if (!errors.IsEmpty)
+ {
+ Assert.Fail("Cross-slice contamination detected:\n" + string.Join("\n", errors));
+ }
+ }
+ finally
+ {
+ foreach (var s in slices) s?.Dispose();
+ }
+ }
+
+ // LUCENENET specific: PR #1267 review item. End-to-end leak gate for the
+ // MMap-specific disposal paths. MockDirectoryWrapper tracks every
+ // directly-opened IndexInput, IndexInputSlicer, and slice and, on Dispose,
+ // throws "cannot close: there are still open files" if any was not disposed.
+ // Wrapping a real MMapDirectory means a missing Dispose on an input, slicer,
+ // or slice fails here. (TestRandomChunkSizes covers the OpenInput-via-
+ // IndexWriter path through MockDirectoryWrapper; this adds explicit coverage
+ // for the slicer/slice paths.)
+ //
+ // Caveats so nobody over-reads this gate:
+ // - Clones are NOT tracked: upstream MockIndexInputWrapper.Clone leaves the
+ // open-file count alone (see the commented LUCENE-686 block there), so a
+ // leaked clone would not fail this test. The clones below verify clone
+ // read behavior, not clone-disposal leakage.
+ // - This gate is at the Lucene IndexInput level: it asserts inputs/slices
+ // are disposed, not that SharedMapping released its backing FileStream -
+ // that lower-level invariant is pinned by
+ // TestDisposeDisposesBackingFileStream_NonEmptyFile.
+ [Test, LuceneNetSpecific]
+ public void TestNoOpenHandlesAfterDispose_SliceAndClonePaths()
+ {
+ var dirPath = CreateTempDir("testMMapNoOpenHandles");
+ var mmapDir = new MMapDirectory(dirPath);
+ // MockDirectoryWrapper takes ownership of mmapDir and disposes it.
+ // dir is NOT in a using: its Dispose() is the assertion under test
+ // (it throws if a handle leaked), so it must run only on the success
+ // path, last. A using would also dispose it while unwinding an earlier
+ // assertion failure, and the resulting "still open files" throw would
+ // mask the real failure. The inputs below ARE in usings: that still
+ // exercises their Dispose() (the path under test) while guaranteeing
+ // cleanup if an assertion in this method throws. usings dispose LIFO,
+ // which gives the correct order (clone before slice before slicer).
+ var dir = new MockDirectoryWrapper(Random, mmapDir);
+
+ const string name = "bytes";
+ using (var io = dir.CreateOutput(name, NewIOContext(Random)))
+ {
+ for (int i = 0; i < 1024; i++) io.WriteInt32(i);
+ }
+
+ // Root input + a clone of it.
+ using (var input = dir.OpenInput(name, NewIOContext(Random)))
+ using (var inputClone = (IndexInput)input.Clone())
+ {
+ Assert.AreEqual(42, ReadInt32At(inputClone, 42));
+ }
+
+ // Slicer + slice + a clone of the slice.
+ using (var slicer = dir.CreateSlicer(name, NewIOContext(Random)))
+ using (var slice = slicer.OpenSlice("half", 0, 1024 * sizeof(int) / 2))
+ using (var sliceClone = (IndexInput)slice.Clone())
+ {
+ Assert.AreEqual(7, ReadInt32At(sliceClone, 7));
+ }
+
+ // If any of the above was left open, this throws
+ // "MockDirectoryWrapper: cannot close: there are still open files".
+ dir.Dispose();
+ }
+
+ private static int ReadInt32At(IndexInput input, long intIndex)
+ {
+ input.Seek(intIndex * sizeof(int));
+ return input.ReadInt32();
+ }
+
+ // Disposing a single slice must not affect its sibling slices from
+ // the same slicer. In the new design each OpenSlice has its own
+ // View, so slice.Dispose closes that slice's view only. Slicer
+ // cascade Dispose is covered by TestCloneSliceSafety.
+ [Test, LuceneNetSpecific]
+ public void TestDisposingOneSliceDoesNotAffectSiblings()
+ {
+ var dirPath = CreateTempDir("testSliceSiblingIsolation");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ for (int i = 0; i < 8; i++) io.WriteInt32(i);
+ }
+
+ using var slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
+ var sliceA = slicer.OpenSlice("sliceA", 0, 16);
+ var sliceB = slicer.OpenSlice("sliceB", 16, 16);
+
+ sliceA.Dispose();
+
+ // Sibling must continue to work and return its own data.
+ Assert.AreEqual(4, sliceB.ReadInt32());
+ Assert.AreEqual(5, sliceB.ReadInt32());
+
+ // Disposed slice must throw on read.
+ try
+ {
+ sliceA.ReadInt32();
+ Assert.Fail("Disposed slice must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+
+ sliceB.Dispose();
+ }
+
+ // Concurrent-clone read correctness. N workers each take a Clone()
+ // of the same root input, seek independently, and read the full
+ // file; every worker must observe the exact bytes that were
+ // written. Existing concurrent tests only check that reads don't
+ // throw — this test pins that readers also don't observe torn or
+ // stale bytes under contention over the shared chunk base pointers.
+ [Test, LuceneNetSpecific, Slow]
+ public void TestConcurrentClonesReadIdenticalBytes()
+ {
+ var dirPath = CreateTempDir("testConcurrentClonesIntegrity");
+ using var mmapDir = new MMapDirectory(dirPath);
+ const string name = "bytes";
+ // 2 MiB — large enough that readers overlap in time, small
+ // enough that the test finishes in well under a second per
+ // iteration.
+ const int fileSize = 2 * 1024 * 1024;
+ var random = Random;
+ var expected = new byte[fileSize];
+ random.NextBytes(expected);
+
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ io.WriteBytes(expected, 0, expected.Length);
+ }
+
+ using var root = mmapDir.OpenInput(name, NewIOContext(random));
+
+ const int numWorkers = 8;
+ const int passesPerWorker = 20;
+ var errors = new ConcurrentBag<string>();
+ var clones = new IndexInput[numWorkers];
+ for (int i = 0; i < numWorkers; i++)
+ {
+ clones[i] = (IndexInput)root.Clone();
+ }
+
+ using var start = new ManualResetEventSlim(false);
+ var threads = new Thread[numWorkers];
+ for (int i = 0; i < numWorkers; i++)
+ {
+ int idx = i;
+ threads[i] = new Thread(() =>
+ {
+ var buf = new byte[fileSize];
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
+ start.Wait();
+ for (int pass = 0; pass < passesPerWorker && errors.IsEmpty; pass++)
+ {
+ clones[idx].Seek(0);
+ clones[idx].ReadBytes(buf, 0, buf.Length);
+ // Byte-equal check: any deviation means the read
+ // path observed torn or stale data.
+ for (int j = 0; j < buf.Length; j++)
+ {
+ if (buf[j] != expected[j])
+ {
+ errors.Add(
+ $"worker {idx} pass {pass}: byte at {j} expected 0x{expected[j]:X2}, got 0x{buf[j]:X2}");
+ return;
+ }
+ }
+ }
+ }) { IsBackground = true, Name = "concurrent-integrity-" + i };
+ threads[i].Start();
+ }
+ start.Set();
+ foreach (var t in threads) t.Join(TimeSpan.FromSeconds(30));
+
+ if (!errors.IsEmpty)
+ {
+ Assert.Fail("Concurrent-clone read corruption detected:\n" + string.Join("\n", errors));
+ }
+ }
+
+ // Open a 3.0 CFS index with MMapDirectory and read it end-to-end
+ // through DirectoryReader. 3.0 (pre-3.1) CFS files are the only
+ // remaining caller of IndexInputSlicer.OpenFullSlice, which in our
+ // new slicer requires a disposed-state check before it can trust
+ // descriptor.Length. If that path is broken, DirectoryReader.Open
+ // will throw while reading segment headers.
+ [Test, LuceneNetSpecific]
+ public void TestRead3xCfsIndex_ViaMMap()
+ {
+ var indexDir = CreateTempDir("test3xCfsIndex");
+ using (var zip = GetType().FindAndGetManifestResourceStream("index.30.cfs.zip"))
+ {
+ Assert.IsNotNull(zip, "expected index.30.cfs.zip to be embedded");
+ TestUtil.Unzip(zip, indexDir);
+ }
+
+ using var mmapDir = new MMapDirectory(indexDir);
+ using var reader = Index.DirectoryReader.Open(mmapDir);
+
+ Assert.IsTrue(reader.MaxDoc > 0, "3.0 index should contain documents");
+
+ // Touch each leaf to force real reads through the CFS +
+ // OpenFullSlice path. This will throw AVE on a broken mmap
+ // teardown or on a bad OpenFullSlice.
+ int totalDocs = 0;
+ foreach (var leaf in reader.Leaves)
+ {
+ var atomic = leaf.AtomicReader;
+ for (int i = 0; i < atomic.MaxDoc; i++)
+ {
+ if (atomic.LiveDocs != null && !atomic.LiveDocs.Get(i))
+ continue;
+ var doc = atomic.Document(i);
+ Assert.IsNotNull(doc, $"doc {i} in leaf {leaf} should not be null");
+ totalDocs++;
+ }
+ }
+ Assert.IsTrue(totalDocs > 0, "at least one live 3.0 document should be readable");
+ }
+
+ // Directly exercise the OpenFullSlice entry point on a 3.0 .cfs
+ // file. OpenFullSlice is [Obsolete("Only for reading CFS files
+ // from 3.x indexes.")] — the test pins that it still produces an
+ // IndexInput spanning the whole file and that its bytes match the
+ // plain OpenInput read of the same file.
+ [Test, LuceneNetSpecific]
+ public void TestOpenFullSlice_On3xCfsFile_MatchesOpenInput()
+ {
+ var indexDir = CreateTempDir("test3xOpenFullSlice");
+ using (var zip = GetType().FindAndGetManifestResourceStream("index.30.cfs.zip"))
+ {
+ Assert.IsNotNull(zip, "expected index.30.cfs.zip to be embedded");
+ TestUtil.Unzip(zip, indexDir);
+ }
+
+ // Pick the first .cfs file in the unzipped 3.0 index.
+ string cfsName = null;
+ foreach (var f in indexDir.GetFiles("*.cfs"))
+ {
+ cfsName = f.Name;
+ break;
+ }
+ Assert.IsNotNull(cfsName, "expected at least one .cfs file in the 3.0 index");
+
+ using var mmapDir = new MMapDirectory(indexDir);
+
+ byte[] viaOpenInput;
+ using (var input = mmapDir.OpenInput(cfsName, NewIOContext(Random)))
+ {
+ viaOpenInput = new byte[input.Length];
+ input.ReadBytes(viaOpenInput, 0, viaOpenInput.Length);
+ }
+
+ byte[] viaFullSlice;
+ using (var slicer = mmapDir.CreateSlicer(cfsName, NewIOContext(Random)))
+ {
+#pragma warning disable 612, 618
+ using var full = slicer.OpenFullSlice();
+#pragma warning restore 612, 618
+ Assert.AreEqual(viaOpenInput.Length, full.Length,
+ "OpenFullSlice length must match OpenInput length");
+ viaFullSlice = new byte[full.Length];
+ full.ReadBytes(viaFullSlice, 0, viaFullSlice.Length);
+ }
+
+ Assert.AreEqual(viaOpenInput, viaFullSlice,
+ "OpenFullSlice bytes must match OpenInput bytes for the same CFS file");
+ }
+
+ // OpenFullSlice on a slicer that has been disposed must throw the
+ // already-closed exception. In Lucene.NET, AlreadyClosedException.Create()
+ // is a factory that returns an ObjectDisposedException (there is no
+ // distinct AlreadyClosedException type), so this test cannot and does not
+ // distinguish the two by type; it asserts via IsAlreadyClosedException(),
+ // which matches the ObjectDisposedException that Create() produces.
+ [Test, LuceneNetSpecific]
+ public void TestOpenFullSlice_AfterDispose_ThrowsAlreadyClosed()
+ {
+ var dirPath = CreateTempDir("testFullSliceAfterDispose");
+ using var mmapDir = new MMapDirectory(dirPath);
+ using (var io = mmapDir.CreateOutput("bytes", NewIOContext(Random)))
+ {
+ io.WriteInt32(42);
+ }
+
+ var slicer = mmapDir.CreateSlicer("bytes", NewIOContext(Random));
+ slicer.Dispose();
+
+ try
+ {
+#pragma warning disable 612, 618
+ slicer.OpenFullSlice();
+#pragma warning restore 612, 618
+ Assert.Fail("OpenFullSlice on disposed slicer must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestDisposeIndexInput()
+ {
+ string name = "foobar";
+ var dir = CreateTempDir("testDisposeIndexInput");
+ string fileName = Path.Combine(dir.FullName, name);
+
+ // Create a zero byte file, and close it immediately
+ File.WriteAllText(fileName, string.Empty, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false) /* No BOM */);
+
+ MMapDirectory mmapDir = new MMapDirectory(dir);
+ using (var _ = mmapDir.OpenInput(name, NewIOContext(Random)))
+ {
+ } // Dispose
+
+ // Now it should be possible to delete the file. This is the condition we are testing for.
+ File.Delete(fileName);
+ }
+
+ // LUCENENET specific: PR #1267 review item. MemoryMappedFile.CreateFromFile
+ // borrows the file handle from the FileStream we pass in but never disposes
+ // the FileStream object itself. SharedMapping therefore owns that FileStream
+ // and must dispose it deterministically on Dispose; otherwise the stream (a
+ // finalizable object holding the file handle) is left to the finalizer. We
+ // assert the invariant directly through internal members rather than by
+ // probing the OS, because the borrowed handle is released either way (the MMF
+ // closes it) and so the leak is not observable as a deletion or exclusive-open
+ // failure. IsFileStreamDisposed reports whether the owned FileStream was disposed.
+ // Note TestDisposeIndexInput above uses a zero-length file, which takes the
+ // early-return path that never calls CreateFromFile and owns no FileStream.
+ [Test, LuceneNetSpecific]
+ public void TestDisposeDisposesBackingFileStream_NonEmptyFile()
+ {
+ const string name = "bytes";
+ var dir = CreateTempDir("testDisposeDisposesBackingFileStream");
+
+ using MMapDirectory mmapDir = new MMapDirectory(dir);
+ using (var output = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ output.WriteInt64(0x0123456789ABCDEFL);
+ }
+
+ var input = (MMapDirectory.MMapIndexInput)mmapDir.OpenInput(name, NewIOContext(Random));
+ var mapping = input.Mapping;
+ Assert.IsFalse(mapping.IsFileStreamDisposed,
+ "backing FileStream must still be open while the input is open");
+
+ input.Dispose();
+
+ Assert.IsTrue(mapping.IsFileStreamDisposed,
+ "disposing the root input must deterministically dispose the mapping's backing FileStream");
+ }
+
+ // LUCENENET specific: PR #1267 review item. The slicer (CreateSlicer) owns its
+ // own SharedMapping; disposing the slicer must dispose that mapping's backing
+ // FileStream just as disposing a root input does. This is the OpenFullSlice /
+ // 3.x CFS path the reviewer called out.
+ [Test, LuceneNetSpecific]
+ public void TestDisposeSlicerDisposesBackingFileStream_NonEmptyFile()
+ {
+ const string name = "bytes";
+ var dir = CreateTempDir("testDisposeSlicerDisposesBackingFileStream");
+
+ using MMapDirectory mmapDir = new MMapDirectory(dir);
+ using (var output = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ output.WriteInt64(0x0123456789ABCDEFL);
+ }
+
+ var slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
+#pragma warning disable 612, 618
+ var full = (MMapDirectory.MMapIndexInput)slicer.OpenFullSlice();
+#pragma warning restore 612, 618
+ var mapping = full.Mapping;
+ Assert.IsFalse(mapping.IsFileStreamDisposed,
+ "backing FileStream must still be open while the slicer is open");
+
+ full.Dispose();
+ slicer.Dispose();
+
+ Assert.IsTrue(mapping.IsFileStreamDisposed,
+ "disposing the slicer must deterministically dispose the mapping's backing FileStream");
+ }
+
+ // LUCENENET specific: tests written to investigate the concern raised
+ // in PR #1267 (review comment r3137038502) that the per-file shared
+ // mapping cache, keyed only by file name with a fixed Length captured
+ // at first-open time, could silently corrupt reads or return wrong
+ // data when a file grows after its mapping is first cached.
+ //
+ // The Lucene directory contract is that a file is write-once: an
+ // IndexOutput is opened, written, closed, and only then may any
+ // IndexInput open it. Once read, the file is never extended by
+ // Lucene — a new commit writes new segment files, it does not append
+ // to existing ones. Readers capture the file length at open time and
+ // never read past it (EOFException otherwise). These tests document
+ // and pin that contract against the current design.
+
+ /// <summary>
+ /// After a reader opens an IndexInput, a concurrent (non-Lucene)
+ /// writer extending the same file must NOT change what the reader
+ /// observes: the IndexInput's Length is a snapshot, reads within
+ /// [0, Length) return the bytes that were present at open time, and
+ /// reads past Length throw EOFException. This is the "snapshot
+ /// length at open time" invariant — the same behavior Java Lucene
+ /// relies on.
+ /// </summary>
+ [Test, LuceneNetSpecific]
+ public void TestGrowthAfterOpen_IsSnapshotAtOpenTime()
+ {
+ var dirPath = CreateTempDir("testGrowthAfterOpen");
+ const string name = "data.bin";
+ string filePath = Path.Combine(dirPath.FullName, name);
+
+ // Seed with a known pattern of 64 bytes.
+ var initial = new byte[64];
+ for (int i = 0; i < initial.Length; i++) initial[i] = (byte)i;
+ File.WriteAllBytes(filePath, initial);
+
+ using var mmapDir = new MMapDirectory(dirPath);
+ using var input = mmapDir.OpenInput(name, NewIOContext(Random));
+
+ Assert.AreEqual(64L, input.Length, "IndexInput.Length must be the snapshot taken at open time.");
+
+ // Extend the file externally by another 64 bytes of different data.
+ using (var fs = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))
+ {
+ var extra = new byte[64];
+ for (int i = 0; i < extra.Length; i++) extra[i] = (byte)(0xFF - i);
+ fs.Write(extra, 0, extra.Length);
+ }
+
+ // Length must not change — the IndexInput is pinned to its
+ // snapshot. This is the "do not track live file length"
+ // contract: the mapping reflects the state at open time.
+ Assert.AreEqual(64L, input.Length,
+ "Growth of the underlying file must NOT be reflected in a previously-opened IndexInput.");
+
+ // Reads within the captured window must return the original bytes.
+ input.Seek(0);
+ var buf = new byte[64];
+ input.ReadBytes(buf, 0, buf.Length);
+ for (int i = 0; i < buf.Length; i++)
+ {
+ Assert.AreEqual((byte)i, buf[i], $"byte[{i}] must be the original value");
+ }
+
+ // Reads past the captured Length must throw EOFException, not
+ // return stale/new bytes. This is what protects Lucene readers
+ // from reading partially-written data in a file that a writer
+ // is still extending.
+ input.Seek(60);
+ Assert.Throws<EndOfStreamException>(() =>
+ {
+ var overflow = new byte[8];
+ input.ReadBytes(overflow, 0, overflow.Length);
+ }, "Reading past the snapshot Length must throw EOFException.");
+ }
+
+ /// <summary>
+ /// If the file has grown on disk between two separate OpenInput
+ /// calls (even for the same file name), the second caller must
+ /// observe the CURRENT file length, not the length cached from
+ /// the first open.
+ /// <para/>
+ /// This is the concern ChatGPT actually articulated in review
+ /// comment r3137038502: a per-file cache keyed only by file name,
+ /// with a fixed Length captured at first mapping time, cannot
+ /// serve a later OpenInput that needs to see bytes the first
+ /// mapping doesn't know about.
+ /// <para/>
+ /// Note: this is NOT a scenario that arises under normal Lucene
+ /// operation — Lucene never extends a segment file once it has
+ /// been closed and referenced by a commit. Lucene writes a new
+ /// file for a new commit. This test documents the edge case for
+ /// non-Lucene callers (and for potential future Lucene behaviors
+ /// that might reuse file names) so that the contract is explicit.
+ /// </summary>
+ [Test, LuceneNetSpecific]
+ public void TestSecondOpenAfterGrowth_ObservesCurrentLength()
+ {
+ var dirPath = CreateTempDir("testSecondOpenAfterGrowth");
+ const string name = "data.bin";
+ string filePath = Path.Combine(dirPath.FullName, name);
+
+ File.WriteAllBytes(filePath, new byte[64]);
+
+ using var mmapDir = new MMapDirectory(dirPath);
+
+ // First open — mapping is created with length=64 and cached.
+ using (var first = mmapDir.OpenInput(name, NewIOContext(Random)))
+ {
+ Assert.AreEqual(64L, first.Length);
+
+ // Grow the file while the first IndexInput is still open.
+ using (var fs = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))
+ {
+ fs.Write(new byte[64], 0, 64);
+ }
+
+ // Second open of the same file, with the first still live,
+ // hits the cached SharedMapping. Under the current design
+ // the cache entry's Length is still 64, so the second
+ // IndexInput sees Length=64 rather than 128.
+ //
+ // This test asserts what we believe the CORRECT behavior
+ // should be. If the current implementation returns 64, the
+ // test fails and we have a real (if Lucene-irrelevant) gap
+ // in the contract to discuss. If it returns 128, the cache
+ // refreshes on length mismatch and no gap exists.
+ using (var second = mmapDir.OpenInput(name, NewIOContext(Random)))
+ {
+ Assert.AreEqual(128L, second.Length,
+ "A second OpenInput after the file grew must see the current file length, " +
+ "not the length cached from the first open. This is the ChatGPT-flagged " +
+ "concern from PR #1267 review r3137038502.");
+ }
+ }
+ }
+
+ /// <summary>
+ /// After the last reference to a cached mapping is dropped, the
+ /// cache entry is removed. A subsequent OpenInput then creates a
+ /// fresh mapping that reflects the current file length. This test
+ /// documents the "cache entry reaped, fresh mapping next time"
+ /// path — the primary mechanism by which the Lucene.NET cache is
+ /// correct under Lucene's actual write-once/read-many semantics.
+ /// </summary>
+ [Test, LuceneNetSpecific]
+ public void TestReopenAfterGrowthWhenCacheDrained_ObservesCurrentLength()
+ {
+ var dirPath = CreateTempDir("testReopenAfterGrowth");
+ const string name = "data.bin";
+ string filePath = Path.Combine(dirPath.FullName, name);
+
+ File.WriteAllBytes(filePath, new byte[64]);
+
+ using var mmapDir = new MMapDirectory(dirPath);
+
+ using (var first = mmapDir.OpenInput(name, NewIOContext(Random)))
+ {
+ Assert.AreEqual(64L, first.Length);
+ } // Dispose drops the last ref, cache entry is reaped.
+
+ // Grow the file after all references are gone.
+ using (var fs = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))
+ {
+ fs.Write(new byte[64], 0, 64);
+ }
+
+ using (var second = mmapDir.OpenInput(name, NewIOContext(Random)))
+ {
+ Assert.AreEqual(128L, second.Length,
+ "After the cache entry was reaped, a new OpenInput must create a fresh " +
+ "mapping reflecting the current file length.");
+ }
+ }
+
+ // LUCENENET specific: regression tests for the direct-pointer
+ // MMapIndexInput. These exercise multi-byte reads that straddle
+ // chunk boundaries, EOF behavior at exact file end, and seek/skip
+ // interactions with the cached chunk pointer.
+
+ private static byte[] WriteFile(MMapDirectory dir, string name, int length)
+ {
+ var bytes = new byte[length];
+ // Deterministic, non-trivial pattern so off-by-one is detectable.
+ for (int i = 0; i < length; i++)
+ {
+ bytes[i] = (byte)((i * 31 + 7) & 0xFF);
+ }
+ using (var io = dir.CreateOutput(name, NewIOContext(Random)))
+ {
+ io.WriteBytes(bytes, 0, bytes.Length);
+ }
+ return bytes;
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadInt16AcrossChunkBoundary()
+ {
+ // chunkSize = 4 bytes; write 8 bytes; the int16 at position 3
+ // starts in chunk 0 (offset 3) and ends in chunk 1 (offset 0).
+ using var mmapDir = new MMapDirectory(CreateTempDir("readInt16AcrossChunk"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 8);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ for (int pos = 0; pos <= bytes.Length - 2; pos++)
+ {
+ ii.Seek(pos);
+ short actual = ii.ReadInt16();
+ short expected = (short)(((bytes[pos] & 0xFF) << 8) | (bytes[pos + 1] & 0xFF));
+ Assert.AreEqual(expected, actual, "ReadInt16 mismatch at position " + pos);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadInt32AcrossChunkBoundary()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("readInt32AcrossChunk"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 16);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ for (int pos = 0; pos <= bytes.Length - 4; pos++)
+ {
+ ii.Seek(pos);
+ int actual = ii.ReadInt32();
+ int expected = ((bytes[pos] & 0xFF) << 24) | ((bytes[pos + 1] & 0xFF) << 16)
+ | ((bytes[pos + 2] & 0xFF) << 8) | (bytes[pos + 3] & 0xFF);
+ Assert.AreEqual(expected, actual, "ReadInt32 mismatch at position " + pos);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadInt64AcrossChunkBoundary()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("readInt64AcrossChunk"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 24);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ for (int pos = 0; pos <= bytes.Length - 8; pos++)
+ {
+ ii.Seek(pos);
+ long actual = ii.ReadInt64();
+ long expected = 0;
+ for (int b = 0; b < 8; b++)
+ {
+ expected = (expected << 8) | (bytes[pos + b] & 0xFFL);
+ }
+ Assert.AreEqual(expected, actual, "ReadInt64 mismatch at position " + pos);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestEofThrowsAtExactLength()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("eofExact"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 7);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+
+ ii.Seek(7);
+ Assert.Throws<EndOfStreamException>(() => ii.ReadByte(), "ReadByte at Length must EOF");
+
+ ii.Seek(6);
+ Assert.Throws<EndOfStreamException>(() => ii.ReadInt16(), "ReadInt16 at Length-1 must EOF");
+
+ ii.Seek(4);
+ Assert.Throws<EndOfStreamException>(() => ii.ReadInt32(), "ReadInt32 at Length-3 must EOF");
+
+ ii.Seek(0);
+ Assert.Throws<EndOfStreamException>(() => ii.ReadInt64(), "ReadInt64 with Length<8 must EOF");
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestBackwardSeekInvalidatesChunkCache()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("backSeek"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 16);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ // Read into chunk 3 to populate cache.
+ ii.Seek(13);
+ Assert.AreEqual(bytes[13], ii.ReadByte());
+ // Backward seek to chunk 0 must invalidate the cache.
+ ii.Seek(1);
+ Assert.AreEqual(bytes[1], ii.ReadByte());
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestSliceWithBaseOffsetAcrossChunkBoundary()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("sliceBaseOffset"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 16);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ // Slice that starts mid-chunk and spans 3 chunk boundaries.
+ using var slice = slicer.OpenSlice("s", 3, 11);
+ Assert.AreEqual(11L, slice.Length);
+ var actual = new byte[11];
+ slice.ReadBytes(actual, 0, 11);
+ for (int i = 0; i < 11; i++)
+ {
+ Assert.AreEqual(bytes[3 + i], actual[i], "slice byte mismatch at " + i);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestSkipBytesAcrossChunks()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("skipBytes"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 32);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ ii.Seek(1);
+ ii.SkipBytes(20);
+ Assert.AreEqual(21L, ii.Position);
+ Assert.AreEqual(bytes[21], ii.ReadByte());
+ // Skip 0 is a no-op.
+ long before = ii.Position;
+ ii.SkipBytes(0);
+ Assert.AreEqual(before, ii.Position);
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCloneIndependentChunkRent()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("cloneIndep"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 32);
+ using var parent = mmapDir.OpenInput("f", NewIOContext(Random));
+ // Parent reads chunk 0.
+ parent.Seek(2);
+ Assert.AreEqual(bytes[2], parent.ReadByte());
+ // Clone: independent position and independent cursor cache.
+ var clone = (IndexInput)parent.Clone();
+ try
+ {
+ clone.Seek(20);
+ Assert.AreEqual(bytes[20], clone.ReadByte());
+ // Parent position is unaffected by clone reads.
+ Assert.AreEqual(3L, parent.Position);
+ Assert.AreEqual(bytes[3], parent.ReadByte());
+ }
+ finally
+ {
+ clone.Dispose();
+ }
+ // Parent still works after clone disposed.
+ Assert.AreEqual(bytes[4], parent.ReadByte());
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestEmptySliceAndEmptyReadBytes()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("emptySlice"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 8);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ using var slice = slicer.OpenSlice("s", 4, 0);
+ Assert.AreEqual(0L, slice.Length);
+ // Empty span read on empty slice is a no-op.
+ slice.ReadBytes(System.Array.Empty<byte>(), 0, 0);
+ // Any non-empty read must EOF.
+ Assert.Throws<EndOfStreamException>(() => slice.ReadByte());
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestPositionTracksReads()
+ {
+ // Verifies Position is accurate after each read primitive.
+ using var mmapDir = new MMapDirectory(CreateTempDir("positionTracks"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 32);
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ Assert.AreEqual(0L, ii.Position);
+ ii.ReadByte(); Assert.AreEqual(1L, ii.Position);
+ ii.ReadInt16(); Assert.AreEqual(3L, ii.Position);
+ ii.ReadInt32(); Assert.AreEqual(7L, ii.Position);
+ ii.ReadInt64(); Assert.AreEqual(15L, ii.Position);
+ ii.SkipBytes(5); Assert.AreEqual(20L, ii.Position);
+ ii.Seek(2); Assert.AreEqual(2L, ii.Position);
+ var buf = new byte[4];
+ ii.ReadBytes(buf, 0, 4);
+ Assert.AreEqual(6L, ii.Position);
+ for (int i = 0; i < 4; i++) Assert.AreEqual(bytes[2 + i], buf[i]);
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadVInt32AcrossChunkBoundary()
+ {
+ // VInt parsing inherits from DataInput and walks via ReadByte;
+ // verify the variable-length encoding survives chunk crossings
+ // without corruption.
+ using var mmapDir = new MMapDirectory(CreateTempDir("readVInt32"), null, 1 << 2);
+ // Write known VInt values that span chunk boundaries.
+ int[] values = { 0, 1, 127, 128, 16383, 16384, int.MaxValue / 2, int.MaxValue };
+ using (var io = mmapDir.CreateOutput("f", NewIOContext(Random)))
+ {
+ foreach (int v in values) io.WriteVInt32(v);
+ }
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ foreach (int expected in values)
+ {
+ Assert.AreEqual(expected, ii.ReadVInt32());
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadVInt64AcrossChunkBoundary()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("readVInt64"), null, 1 << 2);
+ long[] values = { 0L, 1L, 127L, 128L, 16383L, 16384L, long.MaxValue / 2, long.MaxValue };
+ using (var io = mmapDir.CreateOutput("f", NewIOContext(Random)))
+ {
+ foreach (long v in values) io.WriteVInt64(v);
+ }
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ foreach (long expected in values)
+ {
+ Assert.AreEqual(expected, ii.ReadVInt64());
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestReadInt32AcrossChunkBoundary_ColdCache()
+ {
+ // The "cold cache" variant: open, immediately seek to a chunk
+ // seam, ReadInt32. The fast path is only taken if the cached
+ // chunk is already populated, which it isn't on the first call,
+ // so this exercises the slow path explicitly.
+ using var mmapDir = new MMapDirectory(CreateTempDir("readInt32Cold"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 16);
+ for (int pos = 1; pos <= bytes.Length - 4; pos++)
+ {
+ using var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ ii.Seek(pos);
+ int actual = ii.ReadInt32();
+ int expected = ((bytes[pos] & 0xFF) << 24) | ((bytes[pos + 1] & 0xFF) << 16)
+ | ((bytes[pos + 2] & 0xFF) << 8) | (bytes[pos + 3] & 0xFF);
+ Assert.AreEqual(expected, actual, "cold-cache ReadInt32 mismatch at position " + pos);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestDoubleDisposeIsIdempotent()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("doubleDispose"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 16);
+ var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ ii.ReadByte(); // populate the chunk cursor cache
+ ii.Dispose();
+ ii.Dispose(); // must not throw
+ try
+ {
+ ii.ReadByte();
+ Assert.Fail("expected AlreadyClosedException after Dispose");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException() || e is ObjectDisposedException)
+ {
+ // pass
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCrossThreadCloneDisposeWhileReading()
+ {
+ // Regression: clone is being read on thread A; thread B disposes
+ // it. Reader thread must observe close cleanly (AlreadyClosed)
+ // without AVE. This is the targeted same-design analog of the
+ // slicer-vs-slice nightly stress test, but explicit and quick.
+ using var mmapDir = new MMapDirectory(CreateTempDir("xthreadClone"), null, 1 << 4);
+ // 4 KB file, 16-byte chunks → 256 chunks, lots of crossings.
+ WriteFile(mmapDir, "f", 4096);
+ using var parent = mmapDir.OpenInput("f", NewIOContext(Random));
+
+ for (int trial = 0; trial < 20; trial++)
+ {
+ var clone = (IndexInput)parent.Clone();
+ int observedAve = 0;
+ int observedAcceptable = 0;
+
+ var reader = new System.Threading.Thread(() =>
+ {
+ try
+ {
+ clone.Seek(0);
+ long len = clone.Length;
+ long sum = 0;
+ for (long i = 0; i < len; i++) sum += clone.ReadByte();
+ if (sum == long.MinValue) System.Console.WriteLine("never");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException() || e is ObjectDisposedException || e is EndOfStreamException)
+ {
+ System.Threading.Interlocked.Increment(ref observedAcceptable);
+ }
+ catch (System.AccessViolationException)
+ {
+ System.Threading.Interlocked.Increment(ref observedAve);
+ }
+ });
+ reader.Start();
+ System.Threading.Thread.Sleep(0); // let reader start
+ clone.Dispose();
+ reader.Join();
+ Assert.AreEqual(0, observedAve, "AVE observed on trial " + trial);
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCrossThreadCloneDisposeReleasesReadRefDeterministically()
+ {
+ // Regression (#1013): a clone that read on one thread, then is disposed
+ // on a DIFFERENT thread (e.g. a slicer cascade disposing slices other
+ // threads had read), drains cleanly. Because the clone holds no read
+ // bracket open across calls (each read brackets only its own deref via
+ // the DrainReclaimer), once the clone has finished reading it is already
+ // drained, so the owner's subsequent Dispose unmaps every chunk
+ // immediately - no GC/finalizer dependency.
+ //
+ // This test deliberately does NOT call GC.Collect /
+ // WaitForPendingFinalizers anywhere: if cleanup were still
+ // finalizer-dependent, the final assertion would fail.
+ using var mmapDir = new MMapDirectory(CreateTempDir("crossThreadDispose"), null, 1 << 4);
+ // 16-byte chunks, multi-chunk file.
+ WriteFile(mmapDir, "f", 256);
+ var parent = (MMapDirectory.MMapIndexInput)mmapDir.OpenInput("f", NewIOContext(Random));
+ MMapDirectory.Chunk[] chunks = parent.Mapping.Chunks;
+ Assert.IsTrue(chunks.Length > 1, "expected a multi-chunk mapping");
+
+ // A clone reads chunk 0's first byte on a dedicated thread (so the read
+ // happens on a thread other than this one), then this thread disposes
+ // the clone - the cross-thread Dispose path.
+ DisposeCloneCrossThread(parent);
+
+ // The clone finished reading and was disposed (cross-thread), so it is
+ // fully drained. Disposing the root now closes the mapping; with no
+ // reader in-flight the reclaimer unmaps every chunk immediately, with
+ // no finalizer step.
+ parent.Dispose();
+
+ foreach (var c in chunks)
+ {
+ Assert.IsTrue(c.IsNativeReleased,
+ "every chunk accessor must be unmapped synchronously by the " +
+ "root's Dispose; a stranded in-flight reader would have deferred " +
+ "this until it drained");
+ }
+ Assert.IsTrue(parent.Mapping.IsFileStreamDisposed,
+ "the backing FileStream must be disposed deterministically too");
+ }
+
+ // Reads chunk 0's first byte on a fresh clone of `parent` from a
+ // dedicated reader thread (so the read happens on THAT thread), then
+ // disposes the clone from THIS thread - exercising the cross-thread
+ // Dispose path. Static and self-contained so the clone and reader thread
+ // are unreachable once this returns.
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static void DisposeCloneCrossThread(MMapDirectory.MMapIndexInput parent)
+ {
+ IndexInput clone = (IndexInput)parent.Clone();
+ // Read on a different thread via a parameterized start so the clone is
+ // NOT captured into a closure that the Thread object would keep alive.
+ // The Thread is local and gone after return.
+ var reader = new System.Threading.Thread(static state =>
+ {
+ var c = (IndexInput)state!;
+ c.Seek(0);
+ c.ReadByte(); // reads chunk 0 on THIS thread, then drains
+ });
+ reader.Start(clone);
+ reader.Join();
+ // Reader has exited (drained). Dispose from this thread (cross-thread
+ // relative to the reader): no reader is in-flight, so this is clean.
+ clone.Dispose();
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCloneDisposeReleasesViewsDeterministically()
+ {
+ // Regression (#1267, NightOwl888 leak concern): a clone holds no
+ // native resource of its own - it shares the root's mapping and only
+ // brackets each read with the reclaimer. Disposing the clone (on its
+ // own thread, the normal path) must NOT tear down the shared mapping,
+ // and disposing the root must then unmap every chunk view and the
+ // backing FileStream synchronously - with NO GC/finalizer step. This
+ // closes the gap that MockDirectoryWrapper's open-files gate cannot
+ // see: it does not track clones, so a clone leaking a view on its own
+ // Dispose would not fail that gate, but it WOULD leave the file mapped
+ // (on Windows, blocking a later overwrite/delete - exactly
+ // NightOwl888's "Cannot overwrite" symptom). This test never calls
+ // GC.Collect/WaitForPendingFinalizers.
+ using var mmapDir = new MMapDirectory(CreateTempDir("cloneDisposeLeak"), null, 1 << 4);
+ // 16-byte chunks, multi-chunk file.
+ WriteFile(mmapDir, "f", 256);
+ var root = (MMapDirectory.MMapIndexInput)mmapDir.OpenInput("f", NewIOContext(Random));
+ MMapDirectory.Chunk[] chunks = root.Mapping.Chunks;
+ Assert.IsTrue(chunks.Length > 1, "expected a multi-chunk mapping");
+
+ // Read through a clone on THIS thread (crossing a chunk boundary), then
+ // dispose it. The clone is drained the moment each ReadByte returns, so
+ // disposing it must leave the shared mapping untouched.
+ var clone = (IndexInput)root.Clone();
+ clone.Seek(0);
+ for (int i = 0; i < 32; i++) clone.ReadByte(); // crosses chunk 0 -> chunk 1
+ clone.Dispose();
+
+ // The clone is gone. The root still owns the mapping, so chunks remain
+ // mapped (a clone Dispose must NOT tear down the shared mapping).
+ foreach (var c in chunks)
+ {
+ Assert.IsFalse(c.IsNativeReleased,
+ "disposing a clone must NOT unmap the shared chunks the root still owns");
+ }
+ Assert.IsFalse(root.Mapping.IsFileStreamDisposed,
+ "the root still holds the file open after the clone is disposed");
+
+ // The root can still read - proves the clone's Dispose did not disturb
+ // the shared mapping.
+ root.Seek(0);
+ root.ReadByte();
+
+ // Disposing the root unmaps every chunk and the backing FileStream,
+ // synchronously, with no finalizer dependency.
+ root.Dispose();
+ foreach (var c in chunks)
+ {
+ Assert.IsTrue(c.IsNativeReleased,
+ "disposing the root must unmap every chunk view synchronously");
+ }
+ Assert.IsTrue(root.Mapping.IsFileStreamDisposed,
+ "the backing FileStream must be disposed deterministically");
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCloseWhileCloneReadingBlocksUntilDrainThenUnmaps()
+ {
+ // The core teardown guarantee of the DrainReclaimer design, against REAL
+ // mapped memory: when the owner (root) is disposed WHILE a clone is still
+ // inside a read (mid-bracket), Dispose BLOCKS - the chunks stay mapped and
+ // Dispose does not return - until that last reader drains, then unmaps
+ // inline. So teardown is synchronous: by the time Dispose returns, the
+ // views are unmapped. (TestCrossThreadCloneDispose... covers disposing the
+ // clone first; this covers the harder close-while-reading ordering.)
+ //
+ // The reader is parked deterministically inside the bracket via the
+ // existing zero-cost SetOnEnterForTest seam (a null-check already on the
+ // hot path; it adds no production overhead).
+ using var mmapDir = new MMapDirectory(CreateTempDir("closeWhileReading"), null, 1 << 4);
+ WriteFile(mmapDir, "f", 256); // 16-byte chunks, multi-chunk
+ var root = (MMapDirectory.MMapIndexInput)mmapDir.OpenInput("f", NewIOContext(Random));
+ MMapDirectory.Chunk[] chunks = root.Mapping.Chunks;
+ Assert.IsTrue(chunks.Length > 1, "expected a multi-chunk mapping");
+
+ var clone = (MMapDirectory.MMapIndexInput)root.Clone();
+ using var entered = new ManualResetEventSlim(false);
+ using var resume = new ManualResetEventSlim(false);
+ // Park the clone INSIDE its read bracket (admitted, before the load
+ // returns) so the owner's close must observe it as an active reader.
+ clone.SetOnEnterForTest(() =>
+ {
+ // ReSharper disable once AccessToDisposedClosure - runs synchronously
+ entered.Set();
+ // ReSharper disable once AccessToDisposedClosure - runs synchronously
+ resume.Wait();
+ });
+
+ var reader = new Thread(() => { clone.Seek(0); clone.ReadByte(); })
+ { IsBackground = true };
+ reader.Start();
+ Assert.IsTrue(entered.Wait(TimeSpan.FromSeconds(5)),
+ "the clone's reader should park inside the bracket");
+
+ // Dispose the owner while the clone is mid-read: it must BLOCK (spin-wait)
+ // until the reader drains, NOT return, and the chunks must stay mapped.
+ var disposer = new Thread(() => root.Dispose()) { IsBackground = true };
+ disposer.Start();
+ Thread.Sleep(150); // give Dispose time to spin
+ Assert.IsFalse(disposer.Join(TimeSpan.FromMilliseconds(1)),
+ "the owner's Dispose must BLOCK while a reader is inside the bracket");
+ foreach (var c in chunks)
+ {
+ Assert.IsFalse(c.IsNativeReleased,
+ "no chunk may be unmapped while a reader is inside the bracket; " +
+ "unmapping now would free a view under a mid-read clone (AVE)");
+ }
+
+ // Release the reader. Dispose observes the drain, finishes, and unmaps
+ // every chunk inline - so once Dispose returns, the views are GONE.
+ resume.Set();
+ Assert.IsTrue(reader.Join(TimeSpan.FromSeconds(5)), "reader thread should finish");
+ Assert.IsTrue(disposer.Join(TimeSpan.FromSeconds(5)),
+ "Dispose must return once the reader has drained");
+
+ foreach (var c in chunks)
+ {
+ Assert.IsTrue(c.IsNativeReleased,
+ "every chunk must be unmapped synchronously by the time Dispose returns");
+ }
+ Assert.IsTrue(root.Mapping.IsFileStreamDisposed,
+ "the backing FileStream must be disposed synchronously by Dispose");
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestLastSliceFinishedThenSlicerDisposeUnmapsDeterministically()
+ {
+ // The slice variant of the determinism guarantee: a slice reads to
+ // completion (drained, but not yet disposed), then the slicer (the
+ // owner) is disposed - the unmap of every chunk and the FileStream is
+ // immediate and deterministic, with no GC/finalizer step. This is the
+ // "last slice finished reading -> deterministic cleanup" case.
+ using var mmapDir = new MMapDirectory(CreateTempDir("lastSliceDrain"), null, 1 << 4);
+ WriteFile(mmapDir, "f", 256); // 16-byte chunks, multi-chunk
+ var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ var slice = (MMapDirectory.MMapIndexInput)slicer.OpenSlice("s", 0, 256);
+ MMapDirectory.Chunk[] chunks = slice.Mapping.Chunks;
+ Assert.IsTrue(chunks.Length > 1, "expected a multi-chunk mapping");
+
+ // Read the whole slice (crossing every chunk), then the slice is idle
+ // (its bracket is closed after each read - no reference is held open).
+ slice.Seek(0);
+ for (int i = 0; i < 256; i++) slice.ReadByte();
+
+ foreach (var c in chunks)
+ {
+ Assert.IsFalse(c.IsNativeReleased,
+ "chunks stay mapped while the slicer is open");
+ }
+
+ // Dispose the slicer. With the slice idle (drained), Close reclaims
+ // inline: every chunk unmaps and the FileStream closes synchronously.
+ slicer.Dispose();
+ foreach (var c in chunks)
+ {
+ Assert.IsTrue(c.IsNativeReleased,
+ "disposing the slicer after the slice has drained must unmap " +
+ "every chunk synchronously (no finalizer step)");
+ }
+ Assert.IsTrue(slice.Mapping.IsFileStreamDisposed,
+ "the backing FileStream must be disposed deterministically");
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestSlicedReadInt32AcrossOffsets()
+ {
+ // Item 6: exhaustive sweep of slice (offset, length) with
+ // ReadInt32 — exercises the fast-path multi-byte read code with
+ // a non-zero baseOffset that may straddle a chunk boundary at
+ // any offset, including offsets that don't align with any chunk.
+ for (int chunkPower = 0; chunkPower < 5; chunkPower++)
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("slicedReadInt32"), null, 1 << chunkPower);
+ int fileLen = 1 << (chunkPower + 2);
+ var bytes = WriteFile(mmapDir, "f", fileLen);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ for (int sliceStart = 0; sliceStart <= fileLen - 4; sliceStart++)
+ {
+ int maxSliceLen = fileLen - sliceStart;
+ for (int sliceLen = 4; sliceLen <= maxSliceLen; sliceLen++)
+ {
+ using var slice = slicer.OpenSlice("s", sliceStart, sliceLen);
+ for (int innerPos = 0; innerPos <= sliceLen - 4; innerPos++)
+ {
+ slice.Seek(innerPos);
+ int actual = slice.ReadInt32();
+ int abs = sliceStart + innerPos;
+ int expected = ((bytes[abs] & 0xFF) << 24) | ((bytes[abs + 1] & 0xFF) << 16)
+ | ((bytes[abs + 2] & 0xFF) << 8) | (bytes[abs + 3] & 0xFF);
+ Assert.AreEqual(expected, actual,
+ "chunkPower=" + chunkPower + " sliceStart=" + sliceStart +
+ " sliceLen=" + sliceLen + " innerPos=" + innerPos);
+ }
+ }
+ }
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestSlicedReadInt64AcrossOffsets()
+ {
+ for (int chunkPower = 0; chunkPower < 4; chunkPower++)
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("slicedReadInt64"), null, 1 << chunkPower);
+ int fileLen = 1 << (chunkPower + 3);
+ var bytes = WriteFile(mmapDir, "f", fileLen);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ for (int sliceStart = 0; sliceStart <= fileLen - 8; sliceStart++)
+ {
+ int maxSliceLen = fileLen - sliceStart;
+ for (int sliceLen = 8; sliceLen <= maxSliceLen; sliceLen++)
+ {
+ using var slice = slicer.OpenSlice("s", sliceStart, sliceLen);
+ for (int innerPos = 0; innerPos <= sliceLen - 8; innerPos++)
+ {
+ slice.Seek(innerPos);
+ long actual = slice.ReadInt64();
+ int abs = sliceStart + innerPos;
+ long expected = 0L;
+ for (int b = 0; b < 8; b++)
+ expected = (expected << 8) | (bytes[abs + b] & 0xFFL);
+ Assert.AreEqual(expected, actual,
+ "chunkPower=" + chunkPower + " sliceStart=" + sliceStart +
+ " sliceLen=" + sliceLen + " innerPos=" + innerPos);
+ }
+ }
+ }
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestPostXThreadDisposeReadPathsThrow()
+ {
+ // Item 8: after Dispose runs from a different thread (reader is
+ // idle, not mid-deref), every read entry point must observe the
+ // closed state and throw cleanly. Covers ReadByte (fast path
+ // and slow path), ReadInt16/32/64 (fast and slow), ReadBytes,
+ // SkipBytes, and Seek.
+ using var mmapDir = new MMapDirectory(CreateTempDir("postXDispose"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 32);
+
+ void DisposeFromOtherThread(IndexInput target)
+ {
+ var t = new System.Threading.Thread(() => target.Dispose());
+ t.Start();
+ t.Join();
+ }
+
+ // Each scenario opens a fresh input, optionally primes the
+ // chunk cursor cache, disposes from another thread, and verifies
+ // the named operation throws AlreadyClosed on the original
+ // (reader) thread.
+ void Scenario(string name, bool primeCache, Action<IndexInput> op)
+ {
+ var ii = mmapDir.OpenInput("f", NewIOContext(Random));
+ if (primeCache) ii.ReadByte(); // populate currentChunk on the reader thread
+ DisposeFromOtherThread(ii);
+ try
+ {
+ op(ii);
+ Assert.Fail(name + " did not throw after cross-thread Dispose");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException() || e is ObjectDisposedException)
+ {
+ // pass
+ }
+ catch (Exception e)
+ {
+ Assert.Fail(name + " threw unexpected " + e.GetType().Name + ": " + e.Message);
+ }
+ }
+
+ // Cold cache (no chunk cached when Dispose ran).
+ Scenario("ReadByte cold", false, ii => ii.ReadByte());
+ Scenario("ReadInt16 cold", false, ii => ii.ReadInt16());
+ Scenario("ReadInt32 cold", false, ii => ii.ReadInt32());
+ Scenario("ReadInt64 cold", false, ii => ii.ReadInt64());
+ Scenario("ReadBytes cold", false, ii => { var b = new byte[4]; ii.ReadBytes(b, 0, 4); });
+ Scenario("Seek cold", false, ii => ii.Seek(8));
+ Scenario("SkipBytes cold", false, ii => ii.SkipBytes(4));
+
+ // Warm cache (reader had a chunk cached at Dispose time; the reader
+ // observes instanceClosed on its next op and throws, invalidating
+ // its own cursor cache).
+ Scenario("ReadByte warm", true, ii => { for (int i = 0; i < 100; i++) ii.ReadByte(); });
+ Scenario("ReadInt16 warm", true, ii => ii.ReadInt16());
+ Scenario("ReadInt32 warm", true, ii => ii.ReadInt32());
+ Scenario("ReadInt64 warm", true, ii => ii.ReadInt64());
+ Scenario("ReadBytes warm", true, ii => { var b = new byte[16]; ii.ReadBytes(b, 0, 16); });
+ Scenario("Seek warm", true, ii => ii.Seek(8));
+ Scenario("SkipBytes warm", true, ii => ii.SkipBytes(4));
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestOpenSlice_OutOfBounds_Throws()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("sliceBounds"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 16);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+
+ // Negative offset.
+ Assert.Throws<ArgumentOutOfRangeException>(() => slicer.OpenSlice("neg-off", -1, 4));
+ // Negative length.
+ Assert.Throws<ArgumentOutOfRangeException>(() => slicer.OpenSlice("neg-len", 0, -1));
+ // offset + length past end of file.
+ Assert.Throws<ArgumentOutOfRangeException>(() => slicer.OpenSlice("past-end", 8, 10));
+ // offset alone past end of file.
+ Assert.Throws<ArgumentOutOfRangeException>(() => slicer.OpenSlice("off-past-end", 17, 0));
+
+ // Edge: offset == length == 0 is fine on a non-empty file.
+ using (var s = slicer.OpenSlice("empty", 0, 0)) { Assert.AreEqual(0L, s.Length); }
+ // Edge: full file.
+ using (var s = slicer.OpenSlice("full", 0, 16)) { Assert.AreEqual(16L, s.Length); }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestCloneAfterRootDispose_ThrowsAlreadyClosed()
+ {
+ using var mmapDir = new MMapDirectory(CreateTempDir("cloneAfterRootDispose"), null, 1 << 2);
+ WriteFile(mmapDir, "f", 32);
+ var root = mmapDir.OpenInput("f", NewIOContext(Random));
+ root.Dispose();
+ try
+ {
+ root.Clone();
+ Assert.Fail("Clone of disposed root should throw AlreadyClosedException");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException() || e is ObjectDisposedException)
+ {
+ // pass
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void TestSliceNonZeroOffset_SeekToZero_ReadsSliceStart()
+ {
+ // Slice offset lands mid-chunk. After reading some bytes,
+ // Seek(0) must reposition to the slice's first byte (which is
+ // mid-chunk in the underlying file), not to file byte 0.
+ using var mmapDir = new MMapDirectory(CreateTempDir("sliceSeekZero"), null, 1 << 2);
+ var bytes = WriteFile(mmapDir, "f", 32);
+ using var slicer = mmapDir.CreateSlicer("f", NewIOContext(Random));
+ // offset=5 lands inside chunk 1 (chunk size = 4); slice spans
+ // multiple chunks.
+ using var slice = slicer.OpenSlice("s", 5, 20);
+
+ // Drain a few bytes so currentChunk/readBase are populated.
+ for (int i = 0; i < 6; i++)
+ {
+ Assert.AreEqual(bytes[5 + i], slice.ReadByte(), "pre-seek mismatch at " + i);
+ }
+
+ // Seek to slice-relative 0 -> file byte 5.
+ slice.Seek(0);
+ Assert.AreEqual(0L, slice.Position);
+ Assert.AreEqual(bytes[5], slice.ReadByte(), "slice[0] after Seek(0) should equal file[offset]");
+
+ // Read across the slice (covers multiple chunk crossings).
+ slice.Seek(0);
+ var actual = new byte[20];
+ slice.ReadBytes(actual, 0, 20);
+ for (int i = 0; i < 20; i++)
+ {
+ Assert.AreEqual(bytes[5 + i], actual[i], "slice byte mismatch at " + i);
+ }
}
}
}
diff --git a/src/Lucene.Net.Tests/Store/TestUnsafeChunkIndexInput.cs b/src/Lucene.Net.Tests/Store/TestUnsafeChunkIndexInput.cs
new file mode 100644
index 000000000..092a65d19
--- /dev/null
+++ b/src/Lucene.Net.Tests/Store/TestUnsafeChunkIndexInput.cs
@@ -0,0 +1,438 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using Assert = Lucene.Net.TestFramework.Assert;
+
+#nullable enable
+
+namespace Lucene.Net.Store
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET specific: white-box tests for <see cref="UnsafeChunkIndexInput"/>,
+ /// the native read engine shared by <see cref="MMapDirectory"/>. These exercise
+ /// the engine's logic against a managed-<see cref="byte"/>[]-backed chunk source
+ /// (<see cref="ManagedChunkRegion"/>) instead of real memory-mapped views, which
+ /// lets us:
+ /// <list type="bullet">
+ /// <item><description>assert read correctness across chunk boundaries with
+ /// exact expected bytes (a bug is a wrong value, not a process crash);</description></item>
+ /// <item><description>force the fail-fast path (close the region, assert the next
+ /// read throws <see cref="AlreadyClosedException"/>) deterministically;</description></item>
+ /// <item><description>force the cross-thread close path with a reclaimer that
+ /// parks a reader exactly inside the <c>Enter</c>/<c>Exit</c> bracket, and
+ /// assert the close defers the unmap until that reader drains (running it now
+ /// would be an AccessViolation against real memory).</description></item>
+ /// </list>
+ /// A managed array is never freed under a live reader, so these tests cannot
+ /// reproduce a true unmap-vs-read AccessViolationException - that native-safety
+ /// property is covered by the real-mmap stress tests in
+ /// <see cref="TestMultiMMap"/>. Here we test the engine's <i>logic</i>.
+ /// </summary>
+ [TestFixture]
+ [LuceneNetSpecific]
+ public class TestUnsafeChunkIndexInput : LuceneTestCase
+ {
+ // ------------------------------------------------------------------
+ // Managed-byte[]-backed chunk source + input under test
+ // ------------------------------------------------------------------
+
+ /// <summary>
+ /// A managed chunk source: the bytes of a logical region split into
+ /// fixed-size chunks, each backed by a pinned <see cref="byte"/>[]. Shared
+ /// by a root <see cref="ManagedChunkIndexInput"/> and its clones, mirroring
+ /// how a real <c>SharedMapping</c> is shared, including owning an
+ /// <see cref="DrainReclaimer"/> that defers the chunk close until in-flight
+ /// readers drain.
+ /// </summary>
+ internal sealed unsafe class ManagedChunkRegion : IDisposable
+ {
+ internal sealed unsafe class FakeChunk
+ {
+ private readonly byte[] data;
+ private GCHandle handle;
+ internal readonly byte* BasePtr;
+ internal readonly long Length;
+
+ private int closed;
+
+ internal FakeChunk(byte[] data)
+ {
+ this.data = data;
+ this.handle = GCHandle.Alloc(data, GCHandleType.Pinned);
+ this.BasePtr = (byte*)handle.AddrOfPinnedObject();
+ this.Length = data.Length;
+ }
+
+ internal bool IsClosed => Volatile.Read(ref closed) != 0;
+
+ internal void Close()
+ {
+ Volatile.Write(ref closed, 1);
+ }
+
+ internal void FreePin()
+ {
+ if (handle.IsAllocated) handle.Free();
+ }
+ }
+
+ internal readonly FakeChunk[] Chunks;
+ internal readonly long Length;
+ private readonly DrainReclaimer reclaimer = new DrainReclaimer();
+ private int disposed;
+
+ internal DrainReclaimer Reclaimer => reclaimer;
+
+ internal ManagedChunkRegion(byte[] data, int chunkSizePower)
+ {
+ Length = data.Length;
+ long chunkSize = 1L << chunkSizePower;
+ int nChunks = data.Length == 0 ? 0 : (int)((data.Length + chunkSize - 1) >> chunkSizePower);
+ Chunks = new FakeChunk[nChunks];
+ for (int i = 0; i < nChunks; i++)
+ {
+ long off = (long)i << chunkSizePower;
+ int len = (int)Math.Min(chunkSize, data.Length - off);
+ var slice = new byte[len];
+ Array.Copy(data, off, slice, 0, len);
+ Chunks[i] = new FakeChunk(slice);
+ }
+ }
+
+ internal bool IsDisposed => Volatile.Read(ref disposed) != 0;
+
+ // Mirrors SharedMapping.Dispose: closes every chunk through the
+ // reclaimer, which defers the close until in-flight readers drain. Real
+ // teardown of the pin happens in FreeAllPins (test-controlled), so an
+ // AVE is never possible here - managed memory stays valid.
+ public void Dispose()
+ {
+ if (Interlocked.CompareExchange(ref disposed, 1, 0) != 0) return;
+ reclaimer.Close(() =>
+ {
+ foreach (var c in Chunks) c.Close();
+ });
+ }
+
+ internal void FreeAllPins()
+ {
+ foreach (var c in Chunks) c.FreePin();
+ }
+ }
+
+ /// <summary>
+ /// Test <see cref="UnsafeChunkIndexInput"/> subclass backed by a
+ /// <see cref="ManagedChunkRegion"/>. Mirrors <c>MMapDirectory.MMapIndexInput</c>:
+ /// a root owns the region (disposing it closes the chunks); clones share it.
+ /// </summary>
+ internal sealed unsafe class ManagedChunkIndexInput : UnsafeChunkIndexInput
+ {
+ private bool isRoot;
+ private readonly ManagedChunkRegion region;
+
+ internal ManagedChunkRegion Region => region;
+
+ internal ManagedChunkIndexInput(string desc, bool ownsRegion, ManagedChunkRegion region,
+ long offset, long length, int chunkSizePower)
+ : base(region.Reclaimer, desc, offset, length, chunkSizePower)
+ {
+ this.isRoot = ownsRegion;
+ this.region = region;
+ }
+
+ protected override int ChunkCount => region.Chunks.Length;
+
+ protected override byte* ChunkBase(int index) => region.Chunks[index].BasePtr;
+
+ protected override long ChunkLength(int index) => region.Chunks[index].Length;
+
+ public override object Clone()
+ {
+ EnsureOpen();
+ var clone = (ManagedChunkIndexInput)base.Clone();
+ clone.isRoot = false;
+ clone.ResetClonedCursor();
+ return clone;
+ }
+
+ protected override void DisposeChunkSource(bool disposing)
+ {
+ if (isRoot) region.Dispose();
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Helpers
+ // ------------------------------------------------------------------
+
+ private static byte[] MakeData(int len)
+ {
+ var data = new byte[len];
+ for (int i = 0; i < len; i++) data[i] = (byte)(i * 31 + 7);
+ return data;
+ }
+
+ private static ManagedChunkIndexInput OpenRoot(byte[] data, int chunkSizePower, out ManagedChunkRegion region)
+ {
+ region = new ManagedChunkRegion(data, chunkSizePower);
+ return new ManagedChunkIndexInput("root", ownsRegion: true, region, 0, data.Length, chunkSizePower);
+ }
+
+
+ // ------------------------------------------------------------------
+ // Read correctness across chunk boundaries
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestReadByteAcrossChunks()
+ {
+ // 8-byte chunks over a 40-byte region: 5 chunks, every read crosses.
+ var data = MakeData(40);
+ using var input = OpenRoot(data, chunkSizePower: 3, out var region);
+ for (int i = 0; i < data.Length; i++)
+ {
+ Assert.AreEqual(data[i], input.ReadByte(), "byte at " + i);
+ }
+ region.FreeAllPins();
+ }
+
+ [Test]
+ public void TestReadInt32StraddlingChunkBoundary()
+ {
+ // 4-byte chunks: every ReadInt32 except the aligned ones straddles a
+ // boundary, exercising the stackalloc + ReadBytes slow path.
+ var data = MakeData(64);
+ using var input = OpenRoot(data, chunkSizePower: 2, out var region);
+ for (int start = 0; start + 4 <= data.Length; start++)
+ {
+ input.Seek(start);
+ int expected = (data[start] << 24) | (data[start + 1] << 16) | (data[start + 2] << 8) | data[start + 3];
+ Assert.AreEqual(expected, input.ReadInt32(), "Int32 at offset " + start);
+ }
+ region.FreeAllPins();
+ }
+
+ [Test]
+ public void TestReadBytesSpanningMultipleChunks()
+ {
+ var data = MakeData(100);
+ using var input = OpenRoot(data, chunkSizePower: 4, out var region); // 16-byte chunks
+ // Read the whole thing in one ReadBytes call (spans 7 chunks).
+ var buf = new byte[data.Length];
+ input.ReadBytes(buf, 0, buf.Length);
+ Assert.AreEqual(data, buf);
+ // And a sub-range that starts and ends mid-chunk.
+ input.Seek(5);
+ var buf2 = new byte[50];
+ input.ReadBytes(buf2, 0, buf2.Length);
+ for (int i = 0; i < 50; i++) Assert.AreEqual(data[5 + i], buf2[i], "byte " + i);
+ region.FreeAllPins();
+ }
+
+ [Test]
+ public void TestSeekWithinChunkReadsCorrectlyWithoutRecrossing()
+ {
+ var data = MakeData(64);
+ using var input = OpenRoot(data, chunkSizePower: 4, out var region); // 16-byte chunks
+ input.Seek(0);
+ input.ReadByte();
+ // Seek within the same chunk, then across a boundary; both must read the
+ // expected bytes. (The cached base pointer is reused within a chunk and
+ // recomputed on a crossing; correctness is the observable invariant now
+ // that there is no per-crossing native reference to count.)
+ input.Seek(5);
+ Assert.AreEqual(data[5], input.ReadByte());
+ input.Seek(20);
+ Assert.AreEqual(data[20], input.ReadByte());
+ region.FreeAllPins();
+ }
+
+ // ------------------------------------------------------------------
+ // Dispose / region lifecycle
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestDisposeClosesRegionDeterministically()
+ {
+ var data = MakeData(64);
+ var input = OpenRoot(data, chunkSizePower: 4, out var region);
+ input.Seek(0);
+ input.ReadByte();
+
+ input.Dispose(); // same-thread dispose: no in-flight reader, so the
+ // reclaimer reclaims inline and closes the region now.
+ Assert.IsTrue(region.IsDisposed, "disposing the root disposes the region");
+ Assert.IsTrue(region.Chunks[0].IsClosed,
+ "the reclaimer must run the chunk close inline when no reader is active");
+ region.FreeAllPins();
+ }
+
+ [Test]
+ public void TestCloneDisposeLeavesRegionOpenForRoot()
+ {
+ var data = MakeData(64);
+ using var root = OpenRoot(data, chunkSizePower: 4, out var region);
+ root.Seek(0);
+ root.ReadByte();
+ var clone = (IndexInput)root.Clone();
+ clone.Seek(0);
+ clone.ReadByte();
+
+ clone.Dispose(); // a non-owning clone must NOT close the shared region.
+ Assert.IsFalse(region.IsDisposed,
+ "disposing a clone must not dispose the shared region");
+ Assert.IsFalse(region.Chunks[0].IsClosed,
+ "disposing a clone must not close a chunk the root still reads");
+ // Root still works.
+ root.Seek(10);
+ Assert.AreEqual(data[10], root.ReadByte());
+ region.FreeAllPins();
+ }
+
+ // ------------------------------------------------------------------
+ // Fail-fast: read after close throws AlreadyClosed (no crash)
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestCrossingIntoClosedChunkThrowsAlreadyClosed()
+ {
+ var data = MakeData(64);
+ using var root = OpenRoot(data, chunkSizePower: 4, out var region); // 16-byte chunks
+ var clone = (IndexInput)root.Clone();
+ // Read chunk 0 on the clone, then dispose the ROOT (closes all chunks).
+ clone.Seek(0);
+ clone.ReadByte();
+ root.Dispose(); // closes the shared region's chunks
+
+ // The clone is mid-chunk-0; its cached reference is still valid for
+ // chunk 0 reads it already entered... but crossing into chunk 1 must
+ // observe the closed chunk and throw AlreadyClosed (NOT read freed mem,
+ // NOT crash). Seek to chunk 1 to force a crossing.
+ clone.Seek(20);
+ try
+ {
+ clone.ReadByte();
+ Assert.Fail("crossing into a closed chunk must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ region.FreeAllPins();
+ }
+
+ [Test]
+ public void TestReadAfterOwnDisposeThrowsAlreadyClosed()
+ {
+ var data = MakeData(32);
+ var input = OpenRoot(data, chunkSizePower: 3, out var region);
+ input.Seek(0);
+ input.ReadByte();
+ input.Dispose();
+ try
+ {
+ input.ReadByte();
+ Assert.Fail("reading a disposed input must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ region.FreeAllPins();
+ }
+
+ // ------------------------------------------------------------------
+ // Forced cross-thread close: the reclaimer defers the unmap, deterministically
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestCrossThreadCloseDefersUnmapUntilReaderDrains()
+ {
+ // Force the exact interleaving #1013 is about: a reader (thread A) is
+ // INSIDE the reclaimer's Enter/Exit bracket (admitted but parked just
+ // before its dereference) on a clone, while thread B closes the shared
+ // region by disposing the owning root. The reclaimer MUST defer the
+ // actual chunk close (the unmap, against real memory) until A drains -
+ // running it now would free a view under a mid-dereference reader and
+ // AVE. We assert deterministically that the chunk is NOT closed while A
+ // is parked, and IS closed once A exits the bracket.
+ //
+ // True native unmap-vs-read AVE-safety under load is covered against
+ // real mappings by the nightly stress tests in TestMultiMMap; here we
+ // pin down the deferral handshake at the engine's logic level.
+ var data = MakeData(64);
+ using var root = OpenRoot(data, chunkSizePower: 4, out var region);
+ var clone = (ManagedChunkIndexInput)root.Clone();
+
+ var entered = new ManualResetEventSlim(false);
+ var resume = new ManualResetEventSlim(false);
+ // Park the clone's reader INSIDE its Enter/Exit bracket (admitted, before
+ // the dereference returns) so a concurrent Close must wait for it.
+ clone.SetOnEnterForTest(() =>
+ {
+ entered.Set();
+ resume.Wait();
+ });
+
+ var reader = new Thread(state =>
+ {
+ var c = (IndexInput)state!;
+ c.Seek(0);
+ c.ReadByte(); // admitted by Enter, then parks inside the bracket
+ }) { IsBackground = true };
+ reader.Start(clone);
+
+ // Wait until the reader is parked inside the bracket.
+ Assert.IsTrue(entered.Wait(TimeSpan.FromSeconds(5)),
+ "reader should reach the inside of the Enter/Exit bracket");
+
+ // Close the region from THIS (different) thread while the reader is
+ // mid-bracket: the reclaimer must NOT run the unmap yet.
+ var disposer = new Thread(state => ((IDisposable)state!).Dispose())
+ { IsBackground = true };
+ disposer.Start(root);
+
+ // Give the close a chance to (wrongly) reclaim; it must still be parked
+ // because a reader is active. The chunk close is the observable unmap.
+ Thread.Sleep(100);
+ Assert.IsFalse(region.Chunks[0].IsClosed,
+ "a cross-thread close must NOT unmap a chunk while a reader is " +
+ "inside the bracket (against real memory this would AVE)");
+ Assert.IsTrue(region.Reclaimer.IsClosed, "the region was closed (Close was called)");
+
+ // Release the reader: its Exit drains the last reference and runs the
+ // deferred unmap, so the chunk closes now.
+ resume.Set();
+ Assert.IsTrue(reader.Join(TimeSpan.FromSeconds(5)), "reader thread should finish");
+ Assert.IsTrue(disposer.Join(TimeSpan.FromSeconds(5)), "disposer thread should finish");
+
+ // After the reader drains, the deferred unmap has run.
+ for (int i = 0; i < 2000 && !region.Chunks[0].IsClosed; i++) Thread.Sleep(1);
+ Assert.IsTrue(region.Chunks[0].IsClosed,
+ "once the reader exits the bracket, the deferred unmap must run");
+
+ region.FreeAllPins();
+ }
+ }
+}
diff --git a/src/Lucene.Net.Tests/Support/TestApiConsistency.cs b/src/Lucene.Net.Tests/Support/TestApiConsistency.cs
index fff686bc3..00d716c1e 100644
--- a/src/Lucene.Net.Tests/Support/TestApiConsistency.cs
+++ b/src/Lucene.Net.Tests/Support/TestApiConsistency.cs
@@ -38,7 +38,7 @@ public override void TestProtectedFieldNames(Type typeFromTargetAssembly)
[TestCase(typeof(Lucene.Net.Analysis.Analyzer))]
public override void TestPrivateFieldNames(Type typeFromTargetAssembly)
{
- base.TestPrivateFieldNames(typeFromTargetAssembly, @"^Lucene\.Net\.Support\.(?:ConcurrentHashSet|PlatformHelper|DateTimeOffsetUtil|Arrays|IO\.FileSupport)|^Lucene\.ExceptionExtensions|^Lucene\.Net\.Util\.Constants\.MaxStackByteLimit|^Lucene\.Net\.Search\.TopDocs\.ShardByteSize|^Lucene\.Net\.Store\.BaseDirectory\.(?:True|False)|CharStackBufferSize$");
+ base.TestPrivateFieldNames(typeFromTargetAssembly, @"^Lucene\.Net\.Support\.(?:ConcurrentHashSet|PlatformHelper|DateTimeOffsetUtil|Arrays|IO\.FileSupport)|^Lucene\.ExceptionExtensions|^Lucene\.Net\.Util\.Constants\.MaxStackByteLimit|^Lucene\.Net\.Search\.TopDocs\.ShardByteSize|^Lucene\.Net\.Store\.BaseDirectory\.(?:True|False)|^Lucene\.Net\.Store\.MMapDirectory\+Chunk\.(?:BasePtr|Length)$|^Lucene\.Net\.Util\.DrainReclaimer\+Slot\.(?:Depth|OnEnterForTest)$|CharStackBufferSize$");
}
[Test, LuceneNetSpecific]
diff --git a/src/Lucene.Net.Tests/Support/TestDrainReclaimer.cs b/src/Lucene.Net.Tests/Support/TestDrainReclaimer.cs
new file mode 100644
index 000000000..724c8e301
--- /dev/null
+++ b/src/Lucene.Net.Tests/Support/TestDrainReclaimer.cs
@@ -0,0 +1,332 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Assert = Lucene.Net.TestFramework.Assert;
+
+namespace Lucene.Net.Support
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET specific: unit tests for <see cref="DrainReclaimer"/>, the lock-free
+ /// per-user drain barrier that defers a cleanup action until all in-flight users
+ /// have drained. These exercise the handshake at the primitive level, independent
+ /// of <c>MMapDirectory</c> (its real consumer): registration, the
+ /// <c>Enter</c>/<c>Exit</c> bracket and re-entrancy, fail-fast after close, and
+ /// the core invariant that the cleanup runs exactly once and only after every
+ /// active user has drained.
+ /// </summary>
+ [TestFixture]
+ [LuceneNetSpecific]
+ public class TestDrainReclaimer : LuceneTestCase
+ {
+ // ------------------------------------------------------------------
+ // Registration + basic bracket
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestRegisterReturnsDistinctSlots()
+ {
+ var r = new DrainReclaimer();
+ var a = r.Register();
+ var b = r.Register();
+ Assert.IsNotNull(a);
+ Assert.IsNotNull(b);
+ Assert.AreNotSame(a, b, "each Register must return its own slot");
+ }
+
+ [Test]
+ public void TestEnterExitBalancesDepth()
+ {
+ var r = new DrainReclaimer();
+ var slot = r.Register();
+ Assert.AreEqual(0, slot.Depth);
+ slot.EnterCore();
+ Assert.AreEqual(1, slot.Depth, "EnterCore bumps depth");
+ slot.Exit();
+ Assert.AreEqual(0, slot.Depth, "Exit restores depth");
+ }
+
+ [Test]
+ public void TestEnterIsReentrant()
+ {
+ var r = new DrainReclaimer();
+ var slot = r.Register();
+ slot.EnterCore();
+ slot.EnterCore();
+ Assert.AreEqual(2, slot.Depth, "nested Enter increments depth");
+ slot.Exit();
+ Assert.AreEqual(1, slot.Depth, "inner Exit leaves the outer bracket open");
+ slot.Exit();
+ Assert.AreEqual(0, slot.Depth);
+ }
+
+ [Test]
+ public void TestReadScopeUsingEndsBracket()
+ {
+ var r = new DrainReclaimer();
+ var slot = r.Register();
+ using (slot.Enter())
+ {
+ Assert.AreEqual(1, slot.Depth, "the using scope holds the bracket open");
+ }
+ Assert.AreEqual(0, slot.Depth, "disposing the scope ends the bracket");
+ }
+
+ // ------------------------------------------------------------------
+ // Fail-fast after Close
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestEnterAfterCloseThrowsAlreadyClosed()
+ {
+ var r = new DrainReclaimer();
+ var slot = r.Register();
+ r.Close(() => { });
+ Assert.IsTrue(r.IsClosed);
+ try
+ {
+ slot.EnterCore();
+ Assert.Fail("Enter after Close must throw AlreadyClosed");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ Assert.AreEqual(0, slot.Depth, "a rejected Enter must not leave depth elevated");
+ }
+
+ [Test]
+ public void TestSlotRegisteredAfterCloseStillFailsFast()
+ {
+ // Registering a brand-new user after Close is allowed (it just gets a
+ // slot), but its first Enter must observe the closed flag and throw.
+ var r = new DrainReclaimer();
+ r.Close(() => { });
+ var late = r.Register();
+ try
+ {
+ late.EnterCore();
+ Assert.Fail("Enter on a slot registered after Close must throw");
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Cleanup timing: idle vs active
+ // ------------------------------------------------------------------
+
+ [Test]
+ public void TestCloseRunsCleanupImmediatelyWhenIdle()
+ {
+ var r = new DrainReclaimer();
+ r.Register(); // a registered-but-idle user must not block cleanup
+ int cleaned = 0;
+ r.Close(() => cleaned++);
+ Assert.AreEqual(1, cleaned, "with no active user, Close runs the cleanup inline");
+ }
+
+ [Test]
+ public void TestCloseBlocksUntilUserDrainsThenCleansUp()
+ {
+ // A user parked inside the bracket on another thread must BLOCK Close
+ // (which spin-waits until the user drains) and hold off the cleanup; once
+ // the user exits, Close finishes and runs the cleanup inline.
+ var r = new DrainReclaimer();
+ var slot = r.Register();
+
+ var entered = new ManualResetEventSlim(false);
+ var resume = new ManualResetEventSlim(false);
+ slot.OnEnterForTest = () => { entered.Set(); resume.Wait(); };
+
+ int cleaned = 0;
+ var user = new Thread(() =>
+ {
+ slot.EnterCore(); // parks inside the bracket via OnEnterForTest
+ slot.Exit();
+ }) { IsBackground = true };
+ user.Start();
+
+ Assert.IsTrue(entered.Wait(TimeSpan.FromSeconds(5)), "user should park inside the bracket");
+
+ // Close on another thread while the user is active: it must BLOCK (not
+ // return) and must NOT clean up while the user is inside the bracket.
+ var closer = new Thread(() => r.Close(() => Interlocked.Increment(ref cleaned)))
+ { IsBackground = true };
+ closer.Start();
+
+ Thread.Sleep(150); // give Close time to spin
+ Assert.IsFalse(closer.Join(TimeSpan.FromMilliseconds(1)),
+ "Close must block while a user is inside the bracket, not return");
+ Assert.AreEqual(0, Volatile.Read(ref cleaned),
+ "cleanup must not run while a user is inside the bracket");
+
+ // Release the user: Close observes the drain, finishes, and runs cleanup.
+ resume.Set();
+ Assert.IsTrue(user.Join(TimeSpan.FromSeconds(5)), "user thread should finish");
+ Assert.IsTrue(closer.Join(TimeSpan.FromSeconds(5)),
+ "Close must return once the user has drained");
+ Assert.AreEqual(1, Volatile.Read(ref cleaned),
+ "Close runs the cleanup exactly once, synchronously, after the drain");
+ }
+
+ [Test]
+ public void TestCleanupRunsExactlyOnce()
+ {
+ // Many users active at Close; Close blocks until all drain, then fires the
+ // cleanup exactly once.
+ const int users = 8;
+ var r = new DrainReclaimer();
+ var slots = new DrainReclaimer.Slot[users];
+ for (int i = 0; i < users; i++) slots[i] = r.Register();
+
+ int cleaned = 0;
+ var resume = new ManualResetEventSlim(false);
+ var entered = new CountdownEvent(users);
+ var threads = new Thread[users];
+ for (int i = 0; i < users; i++)
+ {
+ var slot = slots[i];
+ slot.OnEnterForTest = () => { entered.Signal(); resume.Wait(); };
+ threads[i] = new Thread(() => { slot.EnterCore(); slot.Exit(); }) { IsBackground = true };
+ threads[i].Start();
+ }
+ Assert.IsTrue(entered.Wait(TimeSpan.FromSeconds(5)), "all users should park");
+
+ var closer = new Thread(() => r.Close(() => Interlocked.Increment(ref cleaned)))
+ { IsBackground = true };
+ closer.Start();
+
+ resume.Set();
+ foreach (var t in threads) Assert.IsTrue(t.Join(TimeSpan.FromSeconds(5)));
+ Assert.IsTrue(closer.Join(TimeSpan.FromSeconds(5)));
+
+ for (int i = 0; i < 2000 && Volatile.Read(ref cleaned) == 0; i++) Thread.Sleep(1);
+ Assert.AreEqual(1, Volatile.Read(ref cleaned), "cleanup must run exactly once");
+ }
+
+ [Test]
+ public void TestSlotsAreIndependent()
+ {
+ // One user being active must not be confused with another being active:
+ // closing while only slot A is active defers; draining A then cleans up,
+ // even though idle slot B was also registered.
+ var r = new DrainReclaimer();
+ var a = r.Register();
+ var b = r.Register();
+ b.EnterCore();
+ b.Exit(); // B is now idle
+
+ a.EnterCore(); // only A active
+ int cleaned = 0;
+ var closer = new Thread(() => r.Close(() => Interlocked.Increment(ref cleaned)))
+ { IsBackground = true };
+ closer.Start();
+ Thread.Sleep(150);
+ Assert.AreEqual(0, Volatile.Read(ref cleaned), "A active -> deferred");
+
+ a.Exit();
+ Assert.IsTrue(closer.Join(TimeSpan.FromSeconds(5)));
+ for (int i = 0; i < 2000 && Volatile.Read(ref cleaned) == 0; i++) Thread.Sleep(1);
+ Assert.AreEqual(1, Volatile.Read(ref cleaned));
+ }
+
+ // ------------------------------------------------------------------
+ // Concurrency stress: never clean up under an active user
+ // ------------------------------------------------------------------
+
+ [Test, LuceneNetSpecific, Slow, Nightly]
+ public void TestConcurrentEnterExitVsCloseNeverCleansUnderActiveUser()
+ {
+ // Hammer the handshake: N users repeatedly Enter/Exit while one thread
+ // Closes at a random moment. The cleanup callback asserts no user is
+ // inside the bracket when it runs (would be the use-after-free analog).
+ // Repeated across many iterations to shake out races.
+ const int iterations = 2000;
+ const int userThreads = 6;
+
+ for (int iter = 0; iter < iterations; iter++)
+ {
+ var r = new DrainReclaimer();
+ var slots = new DrainReclaimer.Slot[userThreads];
+ for (int i = 0; i < userThreads; i++) slots[i] = r.Register();
+
+ int active = 0; // live count of users inside the bracket
+ int violation = 0; // set if cleanup saw active > 0
+ int cleaned = 0;
+ var stop = new ManualResetEventSlim(false);
+
+ var workers = new Task[userThreads];
+ for (int i = 0; i < userThreads; i++)
+ {
+ var slot = slots[i];
+ workers[i] = Task.Run(() =>
+ {
+ try
+ {
+ while (!stop.IsSet)
+ {
+ slot.EnterCore();
+ Interlocked.Increment(ref active);
+ // tiny critical section
+ Interlocked.Decrement(ref active);
+ slot.Exit();
+ }
+ }
+ catch (Exception e) when (e.IsAlreadyClosedException())
+ {
+ // expected once Close wins; EnterCore threw before we
+ // incremented active, so nothing to undo.
+ }
+ });
+ }
+
+ // Let the workers run a moment, then close.
+ Thread.Yield();
+ r.Close(() =>
+ {
+ Interlocked.Increment(ref cleaned);
+ if (Volatile.Read(ref active) != 0)
+ {
+ Interlocked.Exchange(ref violation, 1);
+ }
+ });
+
+ stop.Set();
+ Task.WaitAll(workers);
+
+ Assert.AreEqual(0, Volatile.Read(ref violation),
+ $"iteration {iter}: cleanup ran while a user was inside the bracket");
+ Assert.AreEqual(1, Volatile.Read(ref cleaned),
+ $"iteration {iter}: cleanup must run exactly once");
+ // After everything drains, a final check: no slot left elevated.
+ foreach (var s in slots)
+ {
+ Assert.AreEqual(0, Volatile.Read(ref s.Depth),
+ $"iteration {iter}: a slot was left with depth != 0");
+ }
+ }
+ }
+ }
+}
diff --git a/src/Lucene.Net/Store/MMapDirectory.cs b/src/Lucene.Net/Store/MMapDirectory.cs
index 969d29fdb..6de5f61fa 100644
--- a/src/Lucene.Net/Store/MMapDirectory.cs
+++ b/src/Lucene.Net/Store/MMapDirectory.cs
@@ -1,11 +1,19 @@
-using J2N.IO;
-using J2N.IO.MemoryMappedFiles;
using J2N.Numerics;
using Lucene.Net.Diagnostics;
+using Lucene.Net.Support;
+using Lucene.Net.Support.Threading;
+using Lucene.Net.Util;
+using Microsoft.Win32.SafeHandles;
using System;
using System.IO;
using System.IO.MemoryMappedFiles;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
using System.Threading;
+using SCG = System.Collections.Generic;
+
+#nullable enable
namespace Lucene.Net.Store
{
@@ -26,8 +34,6 @@ namespace Lucene.Net.Store
* limitations under the License.
*/
- using Constants = Lucene.Net.Util.Constants;
-
/// <summary>
/// File-based <see cref="Directory"/> implementation that uses
/// <see cref="MemoryMappedFile"/> for reading, and
@@ -37,9 +43,9 @@ namespace Lucene.Net.Store
/// virtual memory address space in your process equal to the
/// size of the file being mapped. Before using this class,
/// be sure your have plenty of virtual address space, e.g. by
- /// using a 64 bit runtime, or a 32 bit runtime with indexes that are
+ /// using a 64-bit runtime, or a 32-bit runtime with indexes that are
/// guaranteed to fit within the address space.
- /// On 32 bit platforms also consult <see cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/>
+ /// On 32-bit platforms also consult <see cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/>
/// if you have problems with mmap failing because of fragmented
/// address space. If you get an <see cref="OutOfMemoryException"/>, it is recommended
/// to reduce the chunk size, until it works.
@@ -63,15 +69,6 @@ public class MMapDirectory : FSDirectory
private readonly int chunkSizePower;
- // LUCENENET specific BEGIN: test-only counters for the capacity-retry
- // path in Map() — see #1090. Internal (exposed via InternalsVisibleTo
- // to the test assemblies) so regression tests can assert that the
- // race was actually exercised during a run, and to gather data on how
- // many retries are typically needed. Not intended for production use.
- internal static long s_capacityRetryCount;
- internal static int s_maxCapacityAttemptsObserved;
- // LUCENENET specific END
-
/// <summary>
/// Create a new <see cref="MMapDirectory"/> for the named location.
/// </summary>
@@ -79,7 +76,7 @@ public class MMapDirectory : FSDirectory
/// <param name="lockFactory"> the lock factory to use, or null for the default
/// (<see cref="NativeFSLockFactory"/>); </param>
/// <exception cref="IOException"> if there is a low-level I/O error </exception>
- public MMapDirectory(DirectoryInfo path, LockFactory lockFactory)
+ public MMapDirectory(DirectoryInfo path, LockFactory? lockFactory)
: this(path, lockFactory, DEFAULT_MAX_BUFF)
{
}
@@ -114,7 +111,7 @@ public MMapDirectory(DirectoryInfo path)
/// <b>Please note:</b> The chunk size is always rounded down to a power of 2.
/// </param>
/// <exception cref="IOException"> if there is a low-level I/O error </exception>
- public MMapDirectory(DirectoryInfo path, LockFactory lockFactory, int maxChunkSize)
+ public MMapDirectory(DirectoryInfo path, LockFactory? lockFactory, int maxChunkSize)
: base(path, lockFactory)
{
if (maxChunkSize <= 0)
@@ -134,7 +131,7 @@ public MMapDirectory(DirectoryInfo path, LockFactory lockFactory, int maxChunkSi
/// <param name="lockFactory"> the lock factory to use, or null for the default
/// (<see cref="NativeFSLockFactory"/>); </param>
/// <exception cref="IOException"> if there is a low-level I/O error </exception>
- public MMapDirectory(string path, LockFactory lockFactory)
+ public MMapDirectory(string path, LockFactory? lockFactory)
: this(path, lockFactory, DEFAULT_MAX_BUFF)
{
}
@@ -173,17 +170,18 @@ public MMapDirectory(string path)
/// <b>Please note:</b> The chunk size is always rounded down to a power of 2.
/// </param>
/// <exception cref="IOException"> if there is a low-level I/O error </exception>
- public MMapDirectory(string path, LockFactory lockFactory, int maxChunkSize)
+ public MMapDirectory(string path, LockFactory? lockFactory, int maxChunkSize)
: this(new DirectoryInfo(path), lockFactory, maxChunkSize)
{
}
// LUCENENET specific - Some JREs had a bug that didn't allow them to unmap.
// But according to MSDN, the MemoryMappedFile.Dispose() method will
- // indeed "release all resources". Therefore unmap hack is not needed in .NET.
+ // indeed "release all resources". Therefore, unmap hack is not needed in .NET.
/// <summary>
- /// Returns the current mmap chunk size. </summary>
+ /// Returns the current mmap chunk size.
+ /// </summary>
/// <seealso cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/>
public int MaxChunkSize => 1 << chunkSizePower;
@@ -194,40 +192,110 @@ public override IndexInput OpenInput(string name, IOContext context)
EnsureOpen();
EnsureCanRead(name); // LUCENENET-specific: backported call site from Lucene 6.0.0
var file = Path.Combine(Directory.FullName, name); // LUCENENET specific: changed to use string file name instead of allocating a FileInfo (#832)
- var fc = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
- return new MMapIndexInput(this, "MMapIndexInput(path=\"" + file + "\")", fc);
+ // LUCENENET specific: a fresh SharedMapping per OpenInput call, matching
+ // upstream Java (new FileChannel + fc.map()), so Length reflects the
+ // file's current size.
+ SharedMapping mapping = SharedMapping.Create(file, chunkSizePower);
+ // Ownership transfers to the root input (ownsMapping: true); the caller
+ // disposes it, which disposes the mapping. No try/catch around the ctor:
+ // it only sets fields and cannot throw, so the mapping cannot leak here.
+ return new MMapIndexInput($"MMapIndexInput(path=\"{file}\")", ownsMapping: true, mapping, 0, mapping.Length, chunkSizePower);
}
public override IndexInputSlicer CreateSlicer(string name, IOContext context)
{
- // LUCENENET NOTE: name is validated in OpenInput call below
- var full = (MMapIndexInput)OpenInput(name, context);
- return new IndexInputSlicerAnonymousClass(this, full);
+ EnsureOpen();
+ EnsureCanRead(name); // LUCENENET-specific: backported call site from Lucene 6.0.0 (#1357). Unlike upstream, this no longer routes through OpenInput, so validate here too.
+ var file = Path.Combine(Directory.FullName, name);
+ SharedMapping mapping = SharedMapping.Create(file, chunkSizePower);
+ // Ownership transfers to the slicer; the caller disposes it, which
+ // disposes the mapping. No try/catch around the ctor: it only sets
+ // fields and cannot throw, so the mapping cannot leak here.
+ return new IndexInputSlicerAnonymousClass(this, file, mapping);
}
private sealed class IndexInputSlicerAnonymousClass : IndexInputSlicer
{
private readonly MMapDirectory outerInstance;
- private readonly MMapIndexInput full;
- private int disposed = 0; // LUCENENET specific - allow double-dispose
+ private readonly string file;
+
+ // The slicer owns the shared mapping; issued slices piggyback on it.
+ private readonly SharedMapping mapping;
+
+ private int disposed /* = 0 */; // LUCENENET specific - allow double-dispose
- public IndexInputSlicerAnonymousClass(MMapDirectory outerInstance, MMapIndexInput full)
+ // Track issued slices so Dispose cascades: Lucene's contract is that
+ // after slicer.Dispose, reads from any slice (or clone) throw
+ // AlreadyClosedException.
+ private readonly SCG.List<MMapIndexInput> issuedSlices = new SCG.List<MMapIndexInput>();
+ private readonly object issuedSlicesLock = new object();
+
+ public IndexInputSlicerAnonymousClass(MMapDirectory outerInstance, string file, SharedMapping mapping)
{
this.outerInstance = outerInstance;
- this.full = full;
+ this.file = file;
+ this.mapping = mapping;
}
- public override IndexInput OpenSlice(string sliceDescription, long offset, long length)
+ // Returns a slice the CALLER must dispose. The slice does not own the
+ // mapping (ownsMapping: false) but is tracked in issuedSlices so
+ // disposing the slicer cascades to any slices left open. A slice can
+ // outlive a Dispose of outerInstance: we deliberately do not re-check
+ // outerInstance on every read because reads against a disposed mapping
+ // already fail fast with AlreadyClosedException via the reclaimer.
+ // EnsureOpen here only guards the act of opening a new slice.
+ public override IndexInput OpenSlice(string? sliceDescription, long offset, long length)
{
outerInstance.EnsureOpen();
- return full.Slice(sliceDescription, offset, length);
+ // LUCENENET NOTE: TestSeekSliceZero invariant allows 0 offset with 0 length
+ if (offset != 0 && (ulong)offset >= (ulong)mapping.Length)
+ throw new ArgumentOutOfRangeException(nameof(offset),
+ $"slice() {sliceDescription ?? "(null)"} offset out of bounds: " +
+ $"offset={offset},length={length},fileLength={mapping.Length}: {this}");
+ if ((ulong)length > (ulong)mapping.Length)
+ throw new ArgumentOutOfRangeException(nameof(length),
+ $"slice() {sliceDescription ?? "(null)"} length out of bounds: " +
+ $"offset={offset},length={length},fileLength={mapping.Length}: {this}");
+ if ((ulong)offset + (ulong)length > (ulong)mapping.Length)
+ throw new ArgumentOutOfRangeException(
+ $"slice() {sliceDescription ?? "(null)"} parameters out of bounds: " +
+ $"offset={offset},length={length},fileLength={mapping.Length}: {this}");
+
+ // Slices reference the slicer's mapping; only the slicer owns and
+ // disposes it.
+ var input = new MMapIndexInput(
+ $"MMapIndexInput({sliceDescription ?? "(null)"} in path=\"{file}\" slice={offset}..{offset + length})",
+ ownsMapping: false, mapping, offset, length, outerInstance.chunkSizePower);
+
+ UninterruptableMonitor.Enter(issuedSlicesLock);
+ try
+ {
+ if (Volatile.Read(ref disposed) != 0)
+ {
+ // Slicer disposed after EnsureOpen but before we got the
+ // lock; tear down what we just allocated.
+ input.Dispose();
+ throw AlreadyClosedException.Create(nameof(IndexInputSlicer), "this IndexInputSlicer is disposed");
+ }
+
+ issuedSlices.Add(input);
+ }
+ finally
+ {
+ UninterruptableMonitor.Exit(issuedSlicesLock);
+ }
+
+ return input;
}
[Obsolete("Only for reading CFS files from 3.x indexes.")]
public override IndexInput OpenFullSlice()
{
outerInstance.EnsureOpen();
- return (IndexInput)full.Clone();
+ // A full slice is a slice over the whole mapping, sharing the
+ // slicer's single SharedMapping rather than opening a second one.
+ // Length was captured at creation time, so we touch no FileStream.
+ return OpenSlice("full-slice", 0, mapping.Length);
}
protected override void Dispose(bool disposing)
@@ -236,162 +304,425 @@ protected override void Dispose(bool disposing)
if (disposing)
{
- full.Dispose();
+ IDisposable[] toDispose;
+
+ UninterruptableMonitor.Enter(issuedSlicesLock);
+ try
+ {
+ toDispose = issuedSlices.OfType<IDisposable>().ToArray();
+ issuedSlices.Clear();
+ }
+ finally
+ {
+ UninterruptableMonitor.Exit(issuedSlicesLock);
+ }
+
+ IOUtils.DisposeWhileHandlingException(toDispose);
+
+ // Slicer owns the mapping.
+ mapping.Dispose();
}
}
}
- public sealed class MMapIndexInput : ByteBufferIndexInput
+ // LUCENENET specific: the read engine lives in the reusable
+ // Support/UnsafeChunkIndexInput base; this subclass only supplies the
+ // chunk source - the Chunks of a SharedMapping (#1013, #1151).
+ /// <summary>
+ /// LUCENENET-specific <see cref="IndexInput"/> backed by an array of
+ /// memory-mapped <see cref="Chunk"/>s (each a
+ /// <see cref="MemoryMappedViewAccessor"/> whose raw pointer is acquired once
+ /// and cached for the mapping's lifetime). The read logic is inherited from
+ /// <see cref="UnsafeChunkIndexInput"/>; this type wires that engine to a
+ /// <see cref="SharedMapping"/>. The chunked, cached-pointer design
+ /// addresses two Lucene.NET-specific issues together: <b>#1013</b>
+ /// (sporadic <c>AccessViolationException</c> under concurrent search with
+ /// <c>SearcherManager</c>, avoided by the mapping's <see cref="DrainReclaimer"/>
+ /// that defers <c>UnmapViewOfFile</c>/<c>munmap</c> until in-flight reads
+ /// drain), and <b>#1151</b> (MMapDirectory far slower than
+ /// SimpleFSDirectory under parallel load, because
+ /// <c>MemoryMappedViewAccessor</c>'s per-call <c>AcquirePointer</c>/range
+ /// check contends; caching the pointer per chunk removes it).
+ /// </summary>
+ internal sealed unsafe class MMapIndexInput : UnsafeChunkIndexInput
{
- internal MemoryMappedFile memoryMappedFile; // .NET port: this is equivalent to FileChannel.map
- private readonly FileStream fc;
- private int disposed = 0; // LUCENENET specific - allow double-dispose
+ // A "root" instance is one returned from OpenInput; it owns the shared
+ // mapping. Clones and slicer-issued slices are non-root. Non-readonly
+ // so Clone() can clear it on the clone.
+ private bool isRoot;
+
+ // Shared mapping for this IndexInput, shared by the root and its clones,
+ // OR by a slicer + its slices + their clones. Only root instances
+ // (ownsMapping == true) dispose it. This is the only state this subclass
+ // adds; the read engine state lives in the base.
+ private readonly SharedMapping mapping;
+
+ // LUCENENET specific (PR #1267): for testing only. Exposes the shared
+ // mapping so a test can assert that disposing a root input
+ // deterministically disposes the mapping's backing FileStream.
+ internal SharedMapping Mapping => mapping;
- internal MMapIndexInput(MMapDirectory outerInstance, string resourceDescription, FileStream fc)
- : base(resourceDescription, null, fc.Length, outerInstance.chunkSizePower, true)
+ /// <summary>
+ /// Creates an <see cref="IndexInput"/> viewing <c>[offset, offset+length)</c>
+ /// of the given shared mapping. Pass <c>ownsMapping: true</c> only for
+ /// the root returned from <see cref="MMapDirectory.OpenInput(string, IOContext)"/>;
+ /// slices issued from a slicer and clones must pass <c>false</c>.
+ /// </summary>
+ internal MMapIndexInput(string resourceDescription,
+ bool ownsMapping, SharedMapping mapping,
+ long offset, long length, int chunkSizePower)
+ : base(mapping.Reclaimer, resourceDescription, offset, length, chunkSizePower)
{
- this.fc = fc ?? throw new ArgumentNullException(nameof(fc)); // LUCENENET specific - changed from IllegalArgumentException to ArgumentNullException (.NET convention)
- this.SetBuffers(outerInstance.Map(this, fc, 0, fc.Length));
+ this.isRoot = ownsMapping;
+ this.mapping = mapping ?? throw new ArgumentNullException(nameof(mapping));
}
- protected override sealed void Dispose(bool disposing)
+ // --- UnsafeChunkIndexInput chunk source ---------------------------
+
+ protected override int ChunkCount => mapping.Chunks.Length;
+
+ // The chunk caches its base pointer for the mapping's lifetime, so this
+ // is a plain field read; the reclaimer (not a per-crossing acquire)
+ // keeps the view valid against a concurrent close.
+ protected override byte* ChunkBase(int index) => mapping.Chunks[index].BasePtr;
+
+ protected override long ChunkLength(int index) => mapping.Chunks[index].Length;
+
+ // --- Clone / dispose ----------------------------------------------
+
+ public override object Clone()
{
- if (0 != Interlocked.CompareExchange(ref this.disposed, 1, 0)) return; // LUCENENET specific - allow double-dispose
+ // A disposed input must not hand out a working clone.
+ EnsureOpen();
+ // The clone shares the parent's SharedMapping but does NOT own it.
+ // ResetClonedCursor clears the inherited cursor cache and gives the
+ // clone its own reclaimer slot, so the clone and parent read
+ // independently and disposing one never affects the other.
+ var clone = (MMapIndexInput)base.Clone();
+ clone.isRoot = false;
+ clone.ResetClonedCursor();
+ return clone;
+ }
- try
- {
- if (disposing)
- {
- try
- {
- if (this.memoryMappedFile != null)
- {
- this.memoryMappedFile.Dispose();
- this.memoryMappedFile = null;
- }
- }
- finally
- {
- // LUCENENET: If the file is 0 length we will not create a memoryMappedFile above
- // so we must always ensure the FileStream is explicitly disposed.
- this.fc.Dispose();
- }
- }
- }
- finally
+ protected override void DisposeChunkSource(bool disposing)
+ {
+ // Only root instances own the mapping. Disposing it closes the
+ // mapping's reclaimer, which defers the actual unmap of every chunk
+ // until in-flight reads from clones and slices have drained.
+ if (isRoot)
{
- base.Dispose(disposing);
+ mapping.Dispose();
}
}
+ }
+
+
+ /// <summary>
+ /// LUCENENET specific: a single memory-mapped file plus its chunk
+ /// array, owned by exactly one <see cref="MMapIndexInput"/> root or
+ /// one <see cref="Directory.IndexInputSlicer"/>. Clones and slices
+ /// reference it without owning it. The owner calls
+ /// <see cref="SharedMapping.Dispose"/> exactly once; the mapping's
+ /// <see cref="DrainReclaimer"/> defers the actual unmap until in-flight
+ /// reads from non-owning clones/slices have drained, so those reads
+ /// either complete safely against the still-mapped view or fail with
+ /// <see cref="AlreadyClosedException"/> rather than dereferencing a freed
+ /// mapping.
+ /// </summary>
+ internal sealed class SharedMapping : IDisposable
+ {
+ /// <summary>
+ /// The memory-mapped file reference for this mapping.
+ /// Note that this can be null in the edge case of a zero-length mapping.
+ /// </summary>
+ private readonly MemoryMappedFile? memoryMappedFile;
/// <summary>
- /// Try to unmap the buffer, this method silently fails if no support
- /// for that in the runtime. On Windows, this leads to the fact,
- /// that mmapped files cannot be modified or deleted.
+ /// The <see cref="FileStream"/> backing <see cref="memoryMappedFile"/>.
+ /// We pass this stream to
+ /// <see cref="MemoryMappedFile.CreateFromFile(FileStream, string?, long, MemoryMappedFileAccess, HandleInheritability, bool)"/>
+ /// with <c>leaveOpen: true</c>, so the mapping borrows the file handle
+ /// but never disposes the <see cref="FileStream"/> object. This mapping
+ /// owns it and disposes it in <see cref="Dispose"/> so the stream (a
+ /// finalizable object holding the file handle) is released
+ /// deterministically rather than left to the finalizer. Null for the
+ /// zero-length edge case (no mapping is created).
/// </summary>
- protected override void FreeBuffer(ByteBuffer buffer)
- {
- // LUCENENET specific: this should free the memory mapped view accessor
- if (buffer is IDisposable disposable)
- disposable.Dispose();
+ private readonly FileStream? fileStream;
+ private int disposed;
+
+ // The reclaimer that defers this mapping's chunk unmaps until in-flight
+ // readers drain. Shared by every input over this mapping (root, clones,
+ // slices); each registers itself and brackets its reads with it.
+ private readonly DrainReclaimer reclaimer = new DrainReclaimer();
+
+ internal DrainReclaimer Reclaimer => reclaimer;
+
+ // LUCENENET specific (PR #1267): for testing only. True once Dispose has
+ // run and the owned FileStream (if any) has been disposed, so a test can
+ // assert the mapping releases its FileStream deterministically rather
+ // than leaking it to finalization.
+ internal bool IsFileStreamDisposed =>
+ Volatile.Read(ref disposed) != 0 &&
+ (fileStream is null || !fileStream.CanRead);
- // LUCENENET specific: no need for UnmapHack
+ private SharedMapping(MemoryMappedFile? mmf, FileStream? fileStream, Chunk[] chunks, long length)
+ {
+ this.memoryMappedFile = mmf;
+ this.fileStream = fileStream;
+ this.Chunks = chunks;
+ this.Length = length;
}
- }
- /// <summary>
- /// Maps a file into a set of buffers </summary>
- internal virtual ByteBuffer[] Map(MMapIndexInput input, FileStream fc, long offset, long length)
- {
- if ((length >>> chunkSizePower) >= int.MaxValue)
- throw new ArgumentException("RandomAccessFile too big for chunk size: " + fc.ToString());
+ internal static SharedMapping Create(string file, int chunkSizePower)
+ {
+ // .NET Framework's MemoryMappedFile.CreateFromFile reads
+ // fileStream.Length twice non-atomically (defaulting capacity from
+ // it, then enforcing `Length <= capacity`), so a concurrent extender
+ // that grows the file between those reads trips an
+ // ArgumentOutOfRangeException ("capacity"). Modern .NET caches the
+ // length into a single local and reuses it, so the race cannot fire
+ // and this loop runs once. (#1090)
+ // FW: https://github.com/microsoft/referencesource/blob/ec9fa9ae770d522a5b5f0607898044b7478574a3/System.Core/System/IO/MemoryMappedFiles/MemoryMappedFile.cs#L192-L243
+ // modern: https://github.com/dotnet/runtime/blob/550500a978b784658a04110d49b3335dcacf33e0/src/libraries/System.IO.MemoryMappedFiles/src/System/IO/MemoryMappedFiles/MemoryMappedFile.cs#L237-L268
+ // https://github.com/dotnet/runtime/blob/550500a978b784658a04110d49b3335dcacf33e0/src/libraries/System.IO.MemoryMappedFiles/src/System/IO/MemoryMappedFiles/MemoryMappedFile.Windows.cs#L14-L26
+ //
+ // The retry budget is generous because the retry is cheap and a tight
+ // extender can keep winning the race; Yield between attempts so it can
+ // reach a stable point between writes.
+ const int maxAttempts = 32;
+ for (int attempt = 0; ; attempt++)
+ {
+ try
+ {
+ return CreateAttempt(file, chunkSizePower);
+ }
+ catch (ArgumentOutOfRangeException e)
+ when (e.ParamName == "capacity" && attempt < maxAttempts - 1)
+ {
+ // CreateAttempt already disposed its FileStream before
+ // throwing, so just yield and retry.
+ Thread.Yield();
+ }
+ }
+ }
- // LUCENENET specific: Return empty buffer if length is 0, rather than attempting to create a MemoryMappedFile.
- // Part of a solution provided by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
- if (length == 0)
+ private static SharedMapping CreateAttempt(string file, int chunkSizePower)
{
- return new[] { ByteBuffer.Allocate(0).AsReadOnlyBuffer() };
+ // We open our own FileStream to control the FileShare flags. The
+ // path-based CreateFromFile overload uses FileShare.Read, but on
+ // Windows a delete against an open file fails unless the share-mode
+ // permits FILE_SHARE_DELETE, so we need FileShare.Delete (matching
+ // Java's FileChannel default read+write+delete). Without it, callers
+ // that build a temp index then recursively delete it (e.g.
+ // FreeTextSuggester) break.
+ //
+ // bufferSize: 1 because the MMF bypasses the FileStream buffer, so a
+ // 4 KiB default buffer would be allocated and immediately discarded.
+ FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read,
+ FileShare.ReadWrite | FileShare.Delete,
+ bufferSize: 1, FileOptions.RandomAccess);
+ MemoryMappedFile? mmf = null;
+ Chunk[]? chunks = null;
+ try
+ {
+ long length = fs.Length;
+ if (length == 0)
+ {
+ // CreateViewAccessor rejects zero-length views and
+ // CreateFromFile rejects capacity 0, so handle the empty
+ // file ourselves. Dispose fs through the swallowing overload
+ // since no MMF will own it and a throwing Dispose must not
+ // escape this success path.
+ IOUtils.DisposeWhileHandlingException(fs);
+ return new SharedMapping(mmf: null, fileStream: null, chunks: [], length: 0);
+ }
+
+ // capacity: 0 -> the framework sizes the mapping from the file's
+ // current length (the source of the .NET Framework race that
+ // Create's retry loop handles).
+ // leaveOpen: true -> the MMF borrows fs's handle but never
+ // disposes it; SharedMapping owns fs and closes the handle in
+ // Dispose. This keeps a single unambiguous owner of the handle.
+ mmf = MemoryMappedFile.CreateFromFile(
+ fileStream: fs,
+ mapName: null,
+ capacity: 0,
+ access: MemoryMappedFileAccess.Read,
+#if FEATURE_MEMORYMAPPEDFILESECURITY
+ memoryMappedFileSecurity: null,
+#endif
+ inheritability: HandleInheritability.None,
+ leaveOpen: true);
+ chunks = MapChunks(mmf, 0, length, chunkSizePower);
+ return new SharedMapping(mmf, fs, chunks, length);
+ }
+ catch (Exception e) // when (e.IsThrowable())
+ {
+ DisposeResourcesWhileHandlingException(e, chunks, mmf, fs);
+ return null!; // unreachable
+ }
}
- long chunkSize = 1L << chunkSizePower;
+ /// <summary>
+ /// Dispose the underlying native resources. Idempotent. The actual
+ /// unmap is deferred through the reclaimer until in-flight readers
+ /// drain, so it never frees a view a clone or slice is mid-read.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.CompareExchange(ref disposed, 1, 0) != 0) return;
+ reclaimer.Close(() =>
+ {
+ DisposeResourcesWhileHandlingException(null, Chunks, memoryMappedFile, fileStream);
+ });
+ }
- // we always allocate one more buffer, the last one may be a 0 byte one
- int nrBuffers = (int)(length >>> chunkSizePower) + 1;
+ internal Chunk[] Chunks { get; }
- ByteBuffer[] buffers = new ByteBuffer[nrBuffers];
+ internal long Length { get; }
- if (input.memoryMappedFile is null)
+ private static Chunk[] MapChunks(MemoryMappedFile? mmf, long offset, long length, int chunkSizePower)
{
- // LUCENENET specific BEGIN: MemoryMappedFile.CreateFromFile
- // performs an internal stat and throws
- // ArgumentOutOfRangeException("capacity") if the on-disk file
- // size exceeds the requested capacity. When another
- // process/thread is appending to this file (e.g. an
- // IndexWriter that still holds a write handle), the file can
- // grow between when we capture fc.Length and when
- // CreateFromFile reads the size. Retry with the latest
- // observed length on that specific failure; we only map the
- // bytes the caller requested via the buffer-sizing loop
- // below, so an oversized capacity is harmless. See #1090.
- long capacity = Math.Max(length, fc.Length);
- const int maxAttempts = 5;
- int attempt = 0;
- while (true)
+ if (length == 0 || mmf == null)
{
+ return [];
+ }
+
+ long chunkSize = 1L << chunkSizePower;
+ // LUCENENET specific: ceiling-divide for an exact cover with no
+ // trailing empty slot. Upstream Java (MMapDirectory.map) instead
+ // allocates floor + 1 and keeps a final 0-byte sentinel ByteBuffer,
+ // because its read loop unconditionally advances the buffer cursor
+ // past each buffer's end. We don't need that: the base read engine
+ // bounds-checks before indexing and only advances the chunk index
+ // while bytes remain. (CreateViewAccessor rejects a zero-length view
+ // anyway, so a sentinel would need a special case.)
+ int nChunks = (int)((length + chunkSize - 1) >> chunkSizePower);
+ var result = new Chunk[nChunks];
+
+ for (int i = 0; i < nChunks; i++)
+ {
+ long chunkOffset = offset + ((long)i << chunkSizePower);
+ long thisChunkLen = Math.Min(chunkSize, length - ((long)i << chunkSizePower));
+
+ MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(chunkOffset, thisChunkLen, MemoryMappedFileAccess.Read);
+ // The Chunk ctor acquires the view's pointer (a fallible native
+ // call). If it throws, the accessor isn't in result[] yet, so
+ // DisposeChunks in the caller would miss it - dispose it here instead.
try
{
- input.memoryMappedFile = MemoryMappedFile.CreateFromFile(
- fileStream: fc,
- mapName: null,
- capacity: capacity,
- access: MemoryMappedFileAccess.Read,
-#if FEATURE_MEMORYMAPPEDFILESECURITY
- memoryMappedFileSecurity: null,
-#endif
- inheritability: HandleInheritability.Inheritable,
- leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately.
- break;
+ result[i] = new Chunk(accessor, accessor.PointerOffset, thisChunkLen);
}
- catch (ArgumentOutOfRangeException e) when (e.ParamName == "capacity" && attempt < maxAttempts - 1)
+ catch (Exception e)
{
- Interlocked.Increment(ref s_capacityRetryCount);
- capacity = Math.Max(capacity, fc.Length);
- attempt++;
+ IOUtils.DisposeWhileHandlingException(e, accessor);
+ return null!; // unreachable
}
}
- // Record the highest total attempts observed (1 = first try succeeded).
- int attemptsTaken = attempt + 1;
- int prior;
- do
- {
- prior = Volatile.Read(ref s_maxCapacityAttemptsObserved);
- if (attemptsTaken <= prior) break;
- } while (Interlocked.CompareExchange(ref s_maxCapacityAttemptsObserved, attemptsTaken, prior) != prior);
- // LUCENENET specific END
+
+ return result;
}
- long bufferStart = 0L;
- for (int bufNr = 0; bufNr < nrBuffers; bufNr++)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static void DisposeResourcesWhileHandlingException(Exception? priorException, Chunk[]? chunks, MemoryMappedFile? mmf, FileStream? fs)
{
- int bufSize = (int)((length > (bufferStart + chunkSize)) ? chunkSize : (length - bufferStart));
+ // With leaveOpen: true we always own fs; dispose mmf first after chunks so the mapping is torn down
+ // before the backing handle closes.
+ var disposables = ((SCG.IEnumerable<IDisposable?>)(chunks ?? []))
+ .Concat([mmf, fs]);
- // LUCENENET: We get an UnauthorizedAccessException if we create a 0 byte file at the end of the range.
- // See: https://stackoverflow.com/a/5501331
- // We can fix this by using an empty ByteBuffer if the buffer size is 0.
- if (bufSize == 0 && bufNr == (nrBuffers - 1))
+ // DisposeWhileHandlingException tolerates null disposables
+ if (priorException is null)
+ {
+ IOUtils.DisposeWhileHandlingException(disposables);
+ }
+ else
{
- buffers[bufNr] = ByteBuffer.Allocate(0).AsReadOnlyBuffer();
- break;
+ IOUtils.DisposeWhileHandlingException(priorException, disposables);
}
+ }
+ }
- buffers[bufNr] = input.memoryMappedFile.CreateViewByteBuffer(
- offset: offset + bufferStart,
- size: bufSize,
- access: MemoryMappedFileAccess.Read);
- bufferStart += bufSize;
+ /// <summary>
+ /// Owner of a single chunk's <see cref="MemoryMappedViewAccessor"/>.
+ /// One <see cref="Chunk"/> per chunk-sized region of the file; they
+ /// live inside a <see cref="SharedMapping"/>.
+ ///
+ /// <para/>
+ /// Concurrency model: the chunk acquires its raw base pointer ONCE, at
+ /// construction, and caches it for the mapping's whole lifetime. Reads go
+ /// straight to that cached pointer with no per-read or per-crossing
+ /// <see cref="SafeBuffer.AcquirePointer"/> - the per-crossing acquire was
+ /// the #1151 contention, so it is gone. The drain barrier that keeps the
+ /// mapping valid under a concurrent close is now the mapping's
+ /// <see cref="DrainReclaimer"/>: a reader brackets each dereference with
+ /// <c>Enter</c>/<c>Exit</c>, and the recl aimer's <c>Close</c> defers the
+ /// actual <c>UnmapViewOfFile</c>/<c>munmap</c> (this chunk's
+ /// <see cref="Dispose"/>) until every in-flight reader has drained. So an
+ /// AVE is still structurally impossible - the unmap cannot run while a
+ /// reader is mid-dereference - but liveness is proven by the reclaimer's
+ /// hazard handshake rather than by a per-access SafeHandle refcount.
+ /// </summary>
+ internal sealed unsafe class Chunk : IDisposable
+ {
+ private readonly MemoryMappedViewAccessor accessor;
+ private readonly SafeMemoryMappedViewHandle safe;
+ // The cached base pointer, adjusted for the view's page offset. Acquired
+ // once in the ctor and held (one matching ReleasePointer in Release) for
+ // the chunk's whole lifetime, so reads never re-acquire.
+ private readonly byte* basePtr;
+ private bool acquired;
+ internal readonly long Length;
+
+ // Set once Release has disposed the accessor; only makes Release
+ // idempotent. The real teardown synchronization is the reclaimer.
+ private int closed;
+
+ internal Chunk(MemoryMappedViewAccessor accessor, long pointerOffset, long length)
+ {
+ this.accessor = accessor;
+ this.safe = accessor.SafeMemoryMappedViewHandle;
+ byte* ptr = null;
+ safe.AcquirePointer(ref ptr);
+ this.acquired = true;
+ this.basePtr = ptr + pointerOffset;
+ this.Length = length;
}
- return buffers;
+ // LUCENENET specific (PR #1267): for testing only. True once Release has
+ // disposed the accessor, so a test can assert disposing the owning input
+ // tears the chunk down rather than leaking it.
+ internal bool IsNativeReleased => Volatile.Read(ref closed) != 0;
+
+ /// <summary>
+ /// The chunk's cached base pointer (already adjusted for the view's
+ /// page offset). Valid for the chunk's whole lifetime; a reader must
+ /// only dereference it while inside the mapping reclaimer's
+ /// <c>Enter</c>/<c>Exit</c> bracket so a concurrent close cannot unmap
+ /// the view mid-dereference.
+ /// </summary>
+ public byte* BasePtr => basePtr;
+
+ /// <summary>
+ /// Release the cached pointer and dispose this chunk's accessor,
+ /// performing the actual <c>UnmapViewOfFile</c>/<c>munmap</c>. The
+ /// reclaimer only calls this once all in-flight readers have drained,
+ /// so it never unmaps a view out from under a live reader. Idempotent.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.CompareExchange(ref closed, 1, 0) != 0) return;
+ if (acquired)
+ {
+ safe.ReleasePointer();
+ acquired = false;
+ }
+ // Disposing the accessor disposes the SafeHandle; with our matching
+ // ReleasePointer above the refcount reaches zero and the runtime
+ // unmaps. Swallowing overload so a throwing Dispose never propagates.
+ IOUtils.DisposeWhileHandlingException(accessor);
+ }
}
}
}
diff --git a/src/Lucene.Net/Store/NIOFSDirectory.cs b/src/Lucene.Net/Store/NIOFSDirectory.cs
index 341212ee5..1425cdb11 100644
--- a/src/Lucene.Net/Store/NIOFSDirectory.cs
+++ b/src/Lucene.Net/Store/NIOFSDirectory.cs
@@ -104,6 +104,8 @@ public override IndexInput OpenInput(string name, IOContext context)
EnsureOpen();
EnsureCanRead(name); // LUCENENET-specific: backported call site from Lucene 6.0.0
var path = Path.Combine(Directory.FullName, name); // LUCENENET specific: changed to use string file name instead of allocating a FileInfo (#832)
+ // LUCENENET NOTE: FileShare Read+Write+Delete is correct and matches Java Lucene.
+ // In Java, FileChannel defaults to this behavior, and Lucene does not override it.
var fc = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
}
@@ -113,6 +115,8 @@ public override IndexInputSlicer CreateSlicer(string name, IOContext context)
EnsureOpen();
EnsureCanRead(name); // LUCENENET-specific: this method is not in Lucene 6.0.0 but added to match OpenInput above
var path = Path.Combine(Directory.FullName, name); // LUCENENET specific: changed to use string file name instead of allocating a FileInfo (#832)
+ // LUCENENET NOTE: FileShare Read+Write+Delete is correct and matches Java Lucene.
+ // In Java, FileChannel defaults to this behavior, and Lucene does not override it.
var fc = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return new IndexInputSlicerAnonymousClass(context, path, fc);
}
diff --git a/src/Lucene.Net/Support/Store/UnsafeChunkIndexInput.cs b/src/Lucene.Net/Support/Store/UnsafeChunkIndexInput.cs
new file mode 100644
index 000000000..adb8a57fa
--- /dev/null
+++ b/src/Lucene.Net/Support/Store/UnsafeChunkIndexInput.cs
@@ -0,0 +1,566 @@
+using Lucene.Net.Diagnostics;
+using Lucene.Net.Util;
+using System;
+using System.Buffers.Binary;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+#nullable enable
+
+namespace Lucene.Net.Store
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET-specific base class that reads from a fixed-size <i>chunked</i>
+ /// region of unmanaged memory via cached raw pointers, with one bounds check
+ /// per read and no managed-buffer indirection. It encapsulates the native
+ /// read engine that upstream Java keeps in <c>ByteBufferIndexInput</c> (which
+ /// has no direct .NET equivalent), so that a concrete chunk source - e.g.
+ /// <see cref="MMapDirectory"/>'s memory-mapped views - only has to supply the
+ /// chunks; the cursor, the fast read paths, the chunk-crossing logic, and the
+ /// concurrency/teardown coordination all live here and can be reviewed and
+ /// unit-tested independently of any particular chunk source.
+ ///
+ /// <para/>
+ /// There is no upstream counterpart to this class; it lives under
+ /// <c>Support/</c> per the Lucene.NET convention for types with no Java
+ /// equivalent.
+ ///
+ /// <para/>
+ /// <b>Chunk model.</b> The readable region is partitioned into chunks of
+ /// <c>1 << chunkSizePower</c> bytes (the last chunk may be shorter).
+ /// A subclass exposes <see cref="ChunkCount"/> chunks; for each chunk it
+ /// supplies a cached base pointer (<see cref="ChunkBase"/>) and the chunk's
+ /// length (<see cref="ChunkLength"/>). This base never sees the underlying
+ /// chunk implementation (a <c>MemoryMappedViewAccessor</c>, a pinned managed
+ /// array in tests, etc.) - only those two operations.
+ ///
+ /// <para/>
+ /// <b>Window.</b> An instance views <c>[baseOffset, baseOffset + length)</c>
+ /// of the chunked region; this supports slices over a larger backing region.
+ /// All positions exposed to callers are window-relative; chunk lookup
+ /// translates through <c>baseOffset</c>.
+ ///
+ /// <para/>
+ /// <b>Concurrency / teardown.</b> A chunk's base pointer is valid for the whole
+ /// lifetime of the backing region, so the read paths cache it across reads with
+ /// no per-read or per-crossing native call (this is what removes the #1151
+ /// contention). Liveness against a concurrent close is provided by an
+ /// <see cref="DrainReclaimer"/> shared by every instance over a region (a root,
+ /// its clones, and any slices): each instance registers once, and every pointer
+ /// dereference on the read paths is bracketed by the reclaimer's
+ /// <c>Enter</c>/<c>Exit</c>. Closing the region calls <c>Close</c> on the
+ /// reclaimer, which defers the actual unmap until all in-flight readers have
+ /// drained, and makes any later <c>Enter</c> throw <see cref="AlreadyClosedException"/>.
+ /// Because the announce-then-check in <c>Enter</c> is symmetric to the
+ /// publish-then-scan in <c>Close</c>, a reader is never left dereferencing a
+ /// freed view - even when a different thread (a slicer disposing a slice this
+ /// thread is reading) triggers the close. Only the region owner (the root)
+ /// closes the reclaimer; disposing a clone or slice just invalidates that
+ /// instance's cursor.
+ /// </summary>
+ // LUCENENET specific
+ internal abstract unsafe class UnsafeChunkIndexInput : IndexInput
+ {
+ // Per-instance closed flag, independent of per-chunk state, so disposing
+ // a clone does not affect the original or sibling clones.
+ private int instanceClosed;
+
+ // The mapping-wide reclaimer (shared by the root, its clones, and any
+ // slices) and this instance's own reader slot. Each instance must have its
+ // OWN slot: slots carry a per-reader re-entrancy depth, and a clone reads
+ // concurrently with its parent, so they cannot share one. Clone() copies
+ // readerSlot by reference, so ResetClonedCursor re-registers to replace it.
+ // Both are concrete (not an interface) so the hot-path Enter/Exit bracket
+ // inlines with no virtual dispatch.
+ private readonly DrainReclaimer reclaimer;
+ private DrainReclaimer.Slot readerSlot;
+
+ // The window into the chunked region this instance sees (slice range for a
+ // slice, [0, regionLength) for a root). Cached-chunk offsets below are
+ // window-relative; chunk lookup translates via baseOffset.
+ private readonly long baseOffset;
+ private readonly long length;
+ private readonly int chunkSizePower;
+
+ // Window-relative read cursor. 0 <= position <= length.
+ private long position;
+
+ // Cached current-chunk state, valid iff currentChunkIndex >= 0. Holds the
+ // chunk's cached base pointer (from ChunkBase) and is invalidated on a chunk
+ // switch, a Seek to a different chunk, or Dispose. The fast path needs only
+ // readBase and currentEnd beyond `position`; readBase is precomputed so the
+ // load address is a single `*(readBase + pos)`. currentStart is only for Seek
+ // cache validation, not read by ReadByte/ReadInt*.
+ private int currentChunkIndex = NO_CHUNK;
+ private byte* readBase; // = chunkBase + baseOffset - chunkFileStart
+ private long currentStart; // window-relative start of the cached chunk's intersection with [0, length)
+ private long currentEnd; // window-relative end of the cached chunk's intersection with [0, length)
+
+ private const int NO_CHUNK = -1;
+
+ /// <summary>
+ /// Creates an instance viewing <c>[offset, offset + length)</c> of the
+ /// chunked region exposed by the subclass.
+ /// </summary>
+ /// <param name="resourceDescription">A description of the resource, for diagnostics.</param>
+ /// <param name="offset">Window start (window-relative positions are added to this for chunk lookup).</param>
+ /// <param name="length">Window length in bytes.</param>
+ /// <param name="chunkSizePower">log2 of the chunk size; chunks are <c>1 << chunkSizePower</c> bytes (last may be shorter).</param>
+ protected UnsafeChunkIndexInput(DrainReclaimer reclaimer, string resourceDescription,
+ long offset, long length, int chunkSizePower)
+ : base(resourceDescription)
+ {
+ this.reclaimer = reclaimer;
+ this.readerSlot = reclaimer.Register();
+ this.baseOffset = offset;
+ this.length = length;
+ this.chunkSizePower = chunkSizePower;
+ }
+
+ // Test-only: install a callback that this instance's reader slot invokes
+ // INSIDE the Enter/Exit bracket (after admission, before returning to the
+ // read), so a test can park this reader mid-dereference and drive a
+ // concurrent Close. Not on any production path (the slot's hook is null).
+ internal void SetOnEnterForTest(Action onEnter) => readerSlot.OnEnterForTest = onEnter;
+
+ /// <summary>
+ /// Throws <see cref="AlreadyClosedException"/> if this instance has been
+ /// disposed. Subclasses call this from <c>Clone()</c> (a disposed input
+ /// must not hand out a working clone); the base calls it on the read paths.
+ /// </summary>
+ protected void EnsureOpen()
+ {
+ if (Volatile.Read(ref instanceClosed) != 0)
+ {
+ throw AlreadyClosedException.Create(this.GetType().FullName, "Already disposed: " + this);
+ }
+ }
+
+ // --- Subclass seam: the chunk source ----------------------------------
+
+ /// <summary>
+ /// The number of chunks in the backing region. The chunk containing a
+ /// global (non-window-relative) position <c>p</c> is at index
+ /// <c>p >> chunkSizePower</c>, which is always less than this count
+ /// for any in-bounds position.
+ /// </summary>
+ protected abstract int ChunkCount { get; }
+
+ /// <summary>
+ /// The cached base pointer of the chunk at <paramref name="index"/>
+ /// (already adjusted for any page offset). The pointer is valid for the
+ /// chunk's whole lifetime; callers only dereference it inside the
+ /// reclaimer's <c>Enter</c>/<c>Exit</c> bracket so a concurrent close cannot
+ /// unmap the chunk mid-dereference.
+ /// </summary>
+ protected abstract byte* ChunkBase(int index);
+
+ /// <summary>
+ /// The length in bytes of the chunk at <paramref name="index"/> (the last
+ /// chunk may be shorter than <c>1 << chunkSizePower</c>).
+ /// </summary>
+ protected abstract long ChunkLength(int index);
+
+ // --- IndexInput surface ------------------------------------------------
+
+ public override long Length => length;
+
+ public override long Position => position;
+
+ public override void Seek(long pos)
+ {
+ if ((uint)pos > (uint)length)
+ {
+ throw new ArgumentOutOfRangeException(nameof(pos), $"Seek position is out of bounds: {pos}");
+ }
+
+ if (Volatile.Read(ref instanceClosed) != 0)
+ {
+ throw AlreadyClosedException.Create(this.GetType().FullName, "Already disposed: " + this);
+ }
+
+ // If the seek stays inside the cached chunk, keep the cached pointer and
+ // just move the cursor; otherwise invalidate so the next read reacquires.
+ if (currentChunkIndex != NO_CHUNK && pos >= currentStart && pos < currentEnd)
+ {
+ position = pos;
+ return;
+ }
+
+ ReleaseCurrentChunk();
+ position = pos;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public override byte ReadByte()
+ {
+ long pos = position;
+ if (pos < currentEnd)
+ {
+ // EnterCore/Exit bracket the raw load with no try-finally (it inhibits
+ // enregistration); the load can only AVE (uncatchable) and never throws
+ // a managed exception, so Exit is always reached. The other read paths
+ // below follow the same contract - keep the bracketed body throw-free.
+ // See DrainReclaimer.Slot.EnterCore.
+ readerSlot.EnterCore();
+ byte b = *(readBase + pos);
+ readerSlot.Exit();
+ position = pos + 1;
+ return b;
+ }
+ return ReadByteSlow();
+
+ // Slow path: cache miss, EOF, or disposed. Acquires the chunk containing
+ // `position` (or throws), then retries the read.
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ byte ReadByteSlow()
+ {
+ long pos = position;
+ if (pos >= length)
+ {
+ throw EOFException.Create("read past EOF: " + this);
+ }
+ EnsureCurrentChunk(pos);
+
+ readerSlot.EnterCore();
+ byte b = *(readBase + pos);
+ readerSlot.Exit();
+ position = pos + 1;
+ return b;
+ }
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public override short ReadInt16()
+ {
+ long pos = position;
+ if (pos <= currentEnd - 2)
+ {
+ readerSlot.EnterCore();
+ ushort raw = Unsafe.ReadUnaligned<ushort>(readBase + pos);
+ readerSlot.Exit();
+ position = pos + 2;
+ return (short)(BitConverter.IsLittleEndian ? BinaryPrimitives.ReverseEndianness(raw) : raw);
+ }
+
+ return ReadInt16Slow();
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ short ReadInt16Slow()
+ {
+ // Slow path: bytes straddle a chunk boundary. Fill via ReadBytes (which
+ // handles the crossing) and decode big-endian to match the fast path;
+ // avoids the per-byte virtcall round-trip through base.ReadInt16.
+ Span<byte> buf = stackalloc byte[2];
+ ReadBytes(buf);
+ return BinaryPrimitives.ReadInt16BigEndian(buf);
+ }
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public override int ReadInt32()
+ {
+ long pos = position;
+ if (pos <= currentEnd - 4)
+ {
+ readerSlot.EnterCore();
+ uint raw = Unsafe.ReadUnaligned<uint>(readBase + pos);
+ readerSlot.Exit();
+ position = pos + 4;
+ return (int)(BitConverter.IsLittleEndian ? BinaryPrimitives.ReverseEndianness(raw) : raw);
+ }
+
+ return ReadInt32Slow();
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ int ReadInt32Slow()
+ {
+ // Slow path: see ReadInt16.
+ Span<byte> buf = stackalloc byte[4];
+ ReadBytes(buf);
+ return BinaryPrimitives.ReadInt32BigEndian(buf);
+ }
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public override long ReadInt64()
+ {
+ long pos = position;
+ if (pos <= currentEnd - 8)
+ {
+ readerSlot.EnterCore();
+ ulong raw = Unsafe.ReadUnaligned<ulong>(readBase + pos);
+ readerSlot.Exit();
+ position = pos + 8;
+ return (long)(BitConverter.IsLittleEndian ? BinaryPrimitives.ReverseEndianness(raw) : raw);
+ }
+
+ return ReadInt64Slow();
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ long ReadInt64Slow()
+ {
+ // Slow path: see ReadInt16.
+ Span<byte> buf = stackalloc byte[8];
+ ReadBytes(buf);
+ return BinaryPrimitives.ReadInt64BigEndian(buf);
+ }
+ }
+
+ public override void ReadBytes(byte[] b, int offset, int len)
+ {
+ if (b is null)
+ {
+ throw new ArgumentNullException(nameof(b));
+ }
+ if ((uint)offset > (uint)b.Length || (uint)len > (uint)(b.Length - offset))
+ {
+ throw new ArgumentOutOfRangeException(nameof(offset),
+ $"offset/len out of range: offset={offset}, len={len}, b.Length={b.Length}");
+ }
+ if (len == 0) return;
+
+ ReadBytesCore(ref b[offset], len);
+ }
+
+ public override void ReadBytes(Span<byte> destination)
+ {
+ int len = destination.Length;
+ if (len == 0) return;
+
+ ReadBytesCore(ref MemoryMarshal.GetReference(destination), len);
+ }
+
+ // Shared inner loop for both ReadBytes overloads. Takes a raw ref + length
+ // so the byte[] path avoids a Span ctor + GetReference round-trip. The
+ // byte[] overload must bounds-check itself (the Span ctor does it for free).
+ private void ReadBytesCore(ref byte destination, int length)
+ {
+ if (Volatile.Read(ref instanceClosed) != 0)
+ {
+ throw AlreadyClosedException.Create(this.GetType().FullName, "Already disposed: " + this);
+ }
+
+ long pos = position;
+ if ((ulong)pos + (ulong)length > (ulong)this.length)
+ {
+ throw EOFException.Create("read past EOF: " + this);
+ }
+
+ int remaining = length;
+ int dstOff = 0;
+
+ while (remaining > 0)
+ {
+ if (pos >= currentEnd)
+ {
+ EnsureCurrentChunk(pos);
+ }
+
+ long available = currentEnd - pos;
+ int inChunk = (int)(available < remaining ? available : remaining);
+
+ // Bulk copies amortize the bracket over a whole chunk, so unlike the
+ // single-value reads above this path uses the using/ReadScope form: the
+ // try-finally cost is negligible here and the body is larger (more
+ // likely to be edited), so the guaranteed Exit is worth it.
+ using (readerSlot.Enter())
+ {
+ ref byte src = ref Unsafe.AsRef<byte>(readBase + pos);
+ ref byte dst = ref Unsafe.Add(ref destination, dstOff);
+
+ // Small-copy fast path (<= 8 bytes): CopyBlockUnaligned has notable
+ // entry overhead for tiny copies, so a sized read/write is cheaper
+ // for the common short read (e.g. the ReadInt16/Int32/Int64 slow path).
+ if (inChunk <= 8)
+ {
+ switch (inChunk)
+ {
+ case 8:
+ Unsafe.WriteUnaligned(ref dst, Unsafe.ReadUnaligned<ulong>(ref src));
+ break;
+ case 4:
+ Unsafe.WriteUnaligned(ref dst, Unsafe.ReadUnaligned<uint>(ref src));
+ break;
+ case 2:
+ Unsafe.WriteUnaligned(ref dst, Unsafe.ReadUnaligned<ushort>(ref src));
+ break;
+ case 1:
+ dst = src;
+ break;
+ default:
+ // 3, 5, 6, 7 - uncommon; fall through to byte loop.
+ for (int i = 0; i < inChunk; i++)
+ {
+ Unsafe.Add(ref dst, i) = Unsafe.Add(ref src, i);
+ }
+
+ break;
+ }
+ }
+ else
+ {
+ Unsafe.CopyBlockUnaligned(ref dst, ref src, (uint)inChunk);
+ }
+ }
+
+ pos += inChunk;
+ dstOff += inChunk;
+ remaining -= inChunk;
+ }
+ position = pos;
+ }
+
+ public override void SkipBytes(long numBytes)
+ {
+ if (numBytes < 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(numBytes), "numBytes must not be negative");
+ }
+ long newPos = position + numBytes;
+ if ((ulong)newPos >= (ulong)length)
+ {
+ throw EOFException.Create("skip past EOF: " + this);
+ }
+ Seek(newPos);
+ }
+
+ // --- Chunk crossing + teardown ----------------------------------------
+
+ // Point the cursor cache at the chunk containing window-relative
+ // `windowPos`: set readBase / currentStart / currentEnd. Throws
+ // AlreadyClosedException if this instance is closed. No read reference is
+ // taken here - the chunk's base pointer is valid for the mapping's lifetime
+ // and the reclaimer's Enter/Exit (around each dereference) is what keeps a
+ // concurrent close from unmapping it mid-read.
+ private void EnsureCurrentChunk(long windowPos)
+ {
+ if (Volatile.Read(ref instanceClosed) != 0)
+ {
+ throw AlreadyClosedException.Create(this.GetType().FullName, "Already disposed: " + this);
+ }
+ ReleaseCurrentChunk();
+
+ long globalPos = baseOffset + windowPos;
+ int chunkIdx = (int)(globalPos >> chunkSizePower);
+
+ if (Debugging.AssertsEnabled) Debugging.Assert((uint)chunkIdx < (uint)ChunkCount,
+ $"Computed chunk index {chunkIdx} is outside the valid range [0, {ChunkCount}). " +
+ $"windowPos={windowPos}, globalPos={globalPos}, baseOffset={baseOffset}, " +
+ $"length={length}, chunkSizePower={chunkSizePower}");
+
+ byte* chunkBase = ChunkBase(chunkIdx);
+
+ long chunkFileStart = (long)chunkIdx << chunkSizePower;
+ long chunkFileEnd = chunkFileStart + ChunkLength(chunkIdx);
+ // Window-relative interval = chunk's global interval clipped to
+ // [baseOffset, baseOffset + length] and translated.
+ long sliceFileEnd = baseOffset + length;
+ long start = Math.Max(chunkFileStart, baseOffset) - baseOffset;
+ long end = Math.Min(chunkFileEnd, sliceFileEnd) - baseOffset;
+
+ currentChunkIndex = chunkIdx;
+ // Precompute readBase so the fast path is a single load *(readBase + pos).
+ // Address of window byte `pos` is chunkBase + (baseOffset + pos - chunkFileStart)
+ // = (chunkBase + baseOffset - chunkFileStart) + pos.
+ readBase = chunkBase + (baseOffset - chunkFileStart);
+ currentStart = start;
+ currentEnd = end;
+ }
+
+ // Invalidate the cached chunk cursor so the next read reacquires via
+ // EnsureCurrentChunk. There is no native reference to drop (the chunk's
+ // pointer is owned by the mapping and reclaimed only when the mapping
+ // closes), so this just clears the cached fields. Idempotent.
+ private void ReleaseCurrentChunk()
+ {
+ int idx = Interlocked.Exchange(ref currentChunkIndex, NO_CHUNK);
+ if (idx != NO_CHUNK)
+ {
+ readBase = null;
+ currentStart = 0;
+ currentEnd = 0;
+ }
+ }
+
+ // --- Clone / Dispose ---------------------------------------------------
+
+ /// <summary>
+ /// Resets a freshly-cloned instance. Subclasses MUST call this from their
+ /// <c>Clone()</c> override (after <c>base.Clone()</c>) so the clone does not
+ /// share the parent's cached cursor or reader token: it gets its own token
+ /// (its own re-entrancy depth in the reclaimer) and an empty cursor cache,
+ /// so the clone and parent read independently and disposing one never
+ /// affects the other.
+ /// </summary>
+ protected void ResetClonedCursor()
+ {
+ instanceClosed = 0;
+ currentChunkIndex = NO_CHUNK;
+ readBase = null;
+ currentStart = 0;
+ currentEnd = 0;
+ // base.Clone() copied the parent's token by reference; replace it with a
+ // fresh registration so this clone has its own depth slot.
+ readerSlot = reclaimer.Register();
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (!disposing)
+ {
+ return;
+ }
+ if (Interlocked.CompareExchange(ref instanceClosed, 1, 0) != 0)
+ {
+ return;
+ }
+ // Clear currentEnd so a future read's fast path (`pos < currentEnd`)
+ // drops to the slow path and observes instanceClosed. Interlocked so the
+ // 64-bit write can't tear on 32-bit runtimes.
+ //
+ // We deliberately do NOT null readBase here. Disposing an instance while
+ // another thread reads it (e.g. a slicer cascade disposing a slice that
+ // thread is mid-read, or a cross-thread clone Dispose) is racy: that
+ // reader may sit between its `pos < currentEnd` check and its
+ // `*(readBase + pos)` load, so nulling readBase under it would NRE/AVE.
+ // There is no native reference tied to the cursor (the reclaimer owns
+ // reclamation), so leaving readBase stale is harmless - the zeroed
+ // currentEnd already fails-fast every subsequent read, and the in-flight
+ // read completes safely against the still-mapped view (the reclaimer
+ // keeps it mapped until the owning root closes). The fields are GC'd with
+ // the instance.
+ Interlocked.Exchange(ref currentEnd, 0L);
+
+ DisposeChunkSource(disposing);
+ }
+
+ /// <summary>
+ /// Called at the end of <see cref="Dispose(bool)"/> so a subclass can tear
+ /// down the backing chunk source it owns. Only a root input owns the shared
+ /// mapping (and so closes its reclaimer, deferring the unmap until readers
+ /// drain); clones and slices leave this a no-op. The default is a no-op.
+ /// </summary>
+ protected virtual void DisposeChunkSource(bool disposing)
+ {
+ }
+ }
+}
diff --git a/src/Lucene.Net/Support/Util/DrainReclaimer.cs b/src/Lucene.Net/Support/Util/DrainReclaimer.cs
new file mode 100644
index 000000000..469614d3e
--- /dev/null
+++ b/src/Lucene.Net/Support/Util/DrainReclaimer.cs
@@ -0,0 +1,276 @@
+using Lucene.Net.Support.Threading;
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+#nullable enable
+
+namespace Lucene.Net.Util
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET specific: a lock-free reclaimer that runs a cleanup action on a
+ /// shared resource only once every in-flight user has drained, so the cleanup can
+ /// never run while a user is still touching the resource. It is a per-user drain
+ /// barrier (hazard-pointer-inspired, but using a re-entrancy counter per user
+ /// rather than tagging a specific pointer).
+ /// <para/>
+ /// In Lucene.NET it backs <see cref="Store.MMapDirectory"/>: one reclaimer per shared
+ /// memory mapping. <see cref="Close"/> blocks until in-flight reads drain, then
+ /// unmaps - which avoids the #1013 <c>AccessViolationException</c> (a concurrent
+ /// close unmapping a view out from under a reader) without the per-access native
+ /// refcount that caused #1151. It is not tied to memory mapping, though.
+ /// <para/>
+ /// One reclaimer is shared by every user of the resource. Each user calls
+ /// <see cref="Register"/> once to obtain its own <see cref="Slot"/>, then brackets
+ /// each access with <c>using (slot.Enter()) { ... }</c> (or
+ /// <see cref="Slot.EnterCore"/>/<see cref="Slot.Exit"/> directly on hot paths).
+ /// <see cref="Close"/> spin-waits for all in-flight users to drain and then runs
+ /// the cleanup inline, so teardown is synchronous from the caller's point of view.
+ /// <para/>
+ /// The handshake is an announce/scan: <see cref="Slot.EnterCore"/> bumps a
+ /// user-private depth then reads the shared closed flag, and <see cref="Close"/>
+ /// publishes the flag then scans every slot's depth. With asymmetric fencing (see
+ /// <see cref="Slot.EnterCore"/>/<see cref="Close"/>) the user's bracket is
+ /// fence-free on the hot path; the heavy barrier lives only in the rare
+ /// <see cref="Close"/>.
+ /// </summary>
+ internal sealed class DrainReclaimer
+ {
+ /// <summary>
+ /// Per-user state: a re-entrancy depth plus a back-reference to the owning
+ /// reclaimer. One <see cref="Slot"/> per user; a user is single-threaded, so
+ /// only its own thread writes <see cref="Depth"/> (a concurrent
+ /// <see cref="Close"/> scan only reads it).
+ /// </summary>
+ internal sealed class Slot
+ {
+ internal int Depth;
+ private readonly DrainReclaimer owner;
+
+ // Test-only hook: when set, Enter parks here (inside the bracket) so a
+ // test can drive a concurrent Close while this reader is mid-dereference.
+ // Null on every production read; a single predictably-not-taken branch.
+ internal Action? OnEnterForTest;
+
+ internal Slot(DrainReclaimer owner) => this.owner = owner;
+
+ /// <summary>
+ /// Begin a dereference. Returns a <see cref="ReadScope"/> whose
+ /// <c>Dispose</c> ends it; use with <c>using</c>. Throws
+ /// <see cref="AlreadyClosedException"/> if the resource is already closed.
+ /// <para/>
+ /// This is the convenient form, used where the bracket is amortized over a
+ /// bulk operation. The hottest single-value paths instead call
+ /// <see cref="EnterCore"/>/<see cref="Exit"/> directly to skip the
+ /// <c>using</c>/<c>try-finally</c> frame (which inhibits enregistration
+ /// across the protected region); see <see cref="EnterCore"/> for the
+ /// invariant that makes skipping the <c>finally</c> safe there.
+ /// </summary>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public ReadScope Enter()
+ {
+ EnterCore();
+ return new ReadScope(this);
+ }
+
+ /// <summary>
+ /// Begin a dereference without allocating a scope. The caller MUST pair
+ /// this with <see cref="Exit"/>, and - because the hot paths deliberately
+ /// omit a <c>try-finally</c> for speed - the code between the two calls
+ /// MUST be allocation-free and unable to throw a managed exception (a bare
+ /// pointer read/copy qualifies). A true AccessViolation there is
+ /// uncatchable and tears down the process, so a <c>finally</c> would never
+ /// run anyway. A skipped <see cref="Exit"/> would leave <c>Depth</c> stuck
+ /// above zero, which would make a concurrent <see cref="Close"/> spin-wait
+ /// forever (it blocks until every slot drains) - so do NOT add a throwing
+ /// call between <see cref="EnterCore"/> and <see cref="Exit"/>.
+ /// </summary>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void EnterCore()
+ {
+#if FEATURE_INTERLOCKED_MEMORYBARRIERPROCESSWIDE
+ // Asymmetric fencing: NO StoreLoad barrier on the hot path. Announce
+ // with a plain store (this slot is reader-private) then read _closed
+ // plain. The store and read may reorder on this core, but Close issues
+ // a PROCESS-WIDE barrier after setting _closed, so either our Depth
+ // store is visible to Close's scan or our _closed read observes true -
+ // never both passing the check AND staying invisible to the scan.
+ Depth++;
+
+ if (owner._closed)
+ {
+ Depth--;
+ throw AlreadyClosedException.Create(nameof(DrainReclaimer), "Already closed: mapping is closed");
+ }
+#else
+ // No process-wide barrier on this TFM (net462 / netstandard2.0): fall
+ // back to symmetric fencing - a per-read StoreLoad barrier orders the
+ // announce before the _closed check. Slower, but correct everywhere.
+ Volatile.Write(ref Depth, Volatile.Read(ref Depth) + 1);
+ Interlocked.MemoryBarrier();
+
+ if (owner._closed)
+ {
+ Volatile.Write(ref Depth, Volatile.Read(ref Depth) - 1);
+ throw AlreadyClosedException.Create(nameof(DrainReclaimer), "Already closed: mapping is closed");
+ }
+#endif
+ if (OnEnterForTest != null)
+ {
+ OnEnterForTest();
+ }
+ }
+
+ // Ends a dereference (called by ReadScope.Dispose or directly by the hot
+ // read paths). Just publishes the decrement; a concurrent Close blocks
+ // until it observes every slot drained and then unmaps itself, so Exit
+ // never has to run the cleanup.
+ // <para/>
+ // The decrement MUST be a release store. The protected pointer load in the
+ // caller and this Depth-- have no data dependency, so on a weakly-ordered
+ // CPU (ARM64) a plain store could be globally visible BEFORE the load
+ // retires - letting Close's scan see Depth==0 and unmap the page while the
+ // load is still outstanding -> AVE. Volatile.Write is a release barrier:
+ // no preceding memory op (incl. the load) may move after it, so the load
+ // is guaranteed complete before this slot looks drained. (The asymmetric
+ // process-wide barrier in Close only covers the Enter/announce side; the
+ // Exit/retire side needs this release. The net462 fallback already had it.)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void Exit()
+ {
+ Volatile.Write(ref Depth, Depth - 1);
+ }
+ }
+
+ /// <summary>
+ /// A stack-only read bracket. <c>Dispose</c> ends the dereference begun by
+ /// <see cref="Slot.Enter"/>. Being a <c>ref struct</c>, it never heap-allocates
+ /// and its <c>Dispose</c> binds at compile time, so a <c>using</c> over it
+ /// inlines to a plain depth decrement with no interface dispatch.
+ /// </summary>
+ internal readonly ref struct ReadScope
+ {
+ private readonly Slot slot;
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ internal ReadScope(Slot slot) => this.slot = slot;
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void Dispose() => slot.Exit();
+ }
+
+ // Strong refs to every registered reader's slot. Readers (clones, slices)
+ // may be GC'd before Close, but their slot stays here until the reclaimer
+ // itself is collected (it dies with the mapping). A collected reader can't
+ // be mid-dereference, so its slot reads Depth == 0 and the scan skips it;
+ // the retained slots are tiny and bounded by the reader count.
+ private readonly List<Slot> _slots = new();
+
+ private volatile bool _closed;
+
+ public bool IsClosed => _closed;
+
+ /// <summary>
+ /// Register a reader (a root input, a clone, or a slice) and return its slot.
+ /// Each reader registers exactly once, on construction or cloning.
+ /// </summary>
+ public Slot Register()
+ {
+ var slot = new Slot(this);
+ UninterruptableMonitor.Enter(_slots);
+ try
+ {
+ _slots.Add(slot);
+ }
+ finally
+ {
+ UninterruptableMonitor.Exit(_slots);
+ }
+ return slot;
+ }
+
+ /// <summary>
+ /// Close the resource. New <see cref="Slot.Enter"/> calls throw from now on,
+ /// and this BLOCKS (spin-waiting) until every in-flight user has drained, then
+ /// runs the supplied <paramref name="unmap"/> action inline. So when
+ /// <see cref="Close"/> returns, the cleanup has definitely happened - the
+ /// caller (e.g. <c>SharedMapping.Dispose</c>) gets a synchronous teardown.
+ /// <para/>
+ /// This is safe from deadlock under the one-input-per-thread contract: a
+ /// bracket is never held across calls (each read closes its own
+ /// <see cref="Slot.Enter"/>/<see cref="Slot.Exit"/> before returning), so the
+ /// thread that calls <see cref="Close"/> can never itself be holding a bracket
+ /// open. The wait therefore only ever blocks on OTHER threads' short reads,
+ /// the same basis the JVM shared-Arena close relies on.
+ /// </summary>
+ public void Close(Action unmap)
+ {
+ _closed = true;
+ // The one place the heavy synchronization lives. After this barrier, every
+ // reader's prior Depth store is visible to AnyReaderActive below, and every
+ // reader's next _closed read observes true - which is what lets Enter/Exit
+ // run fence-free on the hot path (asymmetric / "biased" synchronization, as
+ // used by hazard-pointer reclamation and the JVM's shared Arena). Without a
+ // process-wide barrier the hot path fences per read instead (see Enter), so
+ // a plain barrier here suffices to pair with it.
+#if FEATURE_INTERLOCKED_MEMORYBARRIERPROCESSWIDE
+ Interlocked.MemoryBarrierProcessWide();
+#else
+ Interlocked.MemoryBarrier();
+#endif
+
+ // Block until all in-flight users drain. SpinOnce escalates from a busy
+ // spin to yielding, so a briefly-descheduled reader (e.g. one that page-
+ // faults on a cold mmap page mid-read) does not pin a core. Reads are
+ // short and bounded (a bracket spans at most one chunk copy), so this
+ // returns promptly in practice.
+ var spin = new SpinWait();
+ while (AnyReaderActive())
+ {
+ spin.SpinOnce();
+ }
+
+ // No user is active and none can newly enter (_closed is published), so
+ // this runs exactly once, here, with no outstanding reader.
+ unmap();
+ }
+
+ private bool AnyReaderActive()
+ {
+ UninterruptableMonitor.Enter(_slots);
+ try
+ {
+ foreach (Slot slot in _slots)
+ {
+ if (Volatile.Read(ref slot.Depth) > 0)
+ {
+ return true;
+ }
+ }
+ }
+ finally
+ {
+ UninterruptableMonitor.Exit(_slots);
+ }
+ return false;
+ }
+ }
+}