You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2022/11/26 12:17:09 UTC
[lucenenet] 03/05: BREAKING: Lucene.Net.Util.OfflineSorter: Refactored to base file tracking on FileStream rather than FileInfo, which gives us better control over temp file deletion by specifying the FileOptions.DeleteOnClose option. We also use random access so we don't need to reopen streams over files except in the case of ExternalRefSorter.
This is an automated email from the ASF dual-hosted git repository.
nightowl888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit 34add98c2559f591814167bd3918821356500025
Author: Shad Storhaug <sh...@shadstorhaug.com>
AuthorDate: Wed Nov 23 20:03:44 2022 +0700
BREAKING: Lucene.Net.Util.OfflineSorter: Refactored to base file tracking on FileStream rather than FileInfo, which gives us better control over temp file deletion by specifying the FileOptions.DeleteOnClose option. We also use random access so we don't need to reopen streams over files except in the case of ExternalRefSorter.
---
.../Analysis/Hunspell/Dictionary.cs | 61 ++---
.../Suggest/Analyzing/AnalyzingSuggester.cs | 20 +-
.../Suggest/Analyzing/FreeTextSuggester.cs | 4 +-
.../Suggest/Fst/ExternalRefSorter.cs | 41 +--
.../Suggest/Fst/FSTCompletionLookup.cs | 13 +-
.../Suggest/SortedInputIterator.cs | 26 +-
.../Suggest/SortedTermFreqIteratorWrapper.cs | 24 +-
.../Analysis/Hunspell/Test64kAffixes.cs | 4 +-
.../Support/TestApiConsistency.cs | 2 +-
src/Lucene.Net.Tests/Util/TestOfflineSorter.cs | 30 ++-
src/Lucene.Net/Support/IO/FileSupport.cs | 101 ++++++-
src/Lucene.Net/Util/OfflineSorter.cs | 292 +++++++++++----------
12 files changed, 358 insertions(+), 260 deletions(-)
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs
index b276ddb98..8a1a24456 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs
@@ -110,7 +110,8 @@ namespace Lucene.Net.Analysis.Hunspell
// when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions
internal bool hasStemExceptions;
- private readonly DirectoryInfo tempDir = OfflineSorter.GetDefaultTempDir(); // TODO: make this configurable?
+ // LUCENENET specific - changed from DirectoryInfo to string
+ private readonly string tempDir = OfflineSorter.DefaultTempDir; // TODO: make this configurable?
internal bool ignoreCase;
internal bool complexPrefixes;
@@ -174,28 +175,20 @@ namespace Lucene.Net.Analysis.Hunspell
this.needsOutputCleaning = false; // set if we have an OCONV
flagLookup.Add(new BytesRef()); // no flags -> ord 0
- FileInfo aff = FileSupport.CreateTempFile("affix", "aff", tempDir);
+ FileStream aff = FileSupport.CreateTempFileAsStream("affix", "aff", tempDir);
try
{
- using (Stream @out = aff.Open(FileMode.Open, FileAccess.ReadWrite))
- {
- // copy contents of affix stream to temp file
- affix.CopyTo(@out);
- }
+ // copy contents of affix stream to temp file
+ affix.CopyTo(aff);
+ aff.Position = 0; // LUCENENET specific - seek to the beginning of the file so we don't need to reopen
// pass 1: get encoding
- string encoding;
- using (Stream aff1 = aff.Open(FileMode.Open, FileAccess.Read))
- {
- encoding = GetDictionaryEncoding(aff1);
- }
+ string encoding = GetDictionaryEncoding(aff);
+ aff.Position = 0; // LUCENENET specific - seek to the beginning of the file so we don't need to reopen
// pass 2: parse affixes
Encoding decoder = GetSystemEncoding(encoding);
- using (Stream aff2 = aff.Open(FileMode.Open, FileAccess.Read))
- {
- ReadAffixFile(aff2, decoder);
- }
+ ReadAffixFile(aff, decoder);
// read dictionary entries
Int32SequenceOutputs o = Int32SequenceOutputs.Singleton;
@@ -207,14 +200,7 @@ namespace Lucene.Net.Analysis.Hunspell
}
finally
{
- try
- {
- aff.Delete();
- }
- catch
- {
- // ignore
- }
+ aff.Dispose();
}
}
@@ -921,8 +907,8 @@ namespace Lucene.Net.Analysis.Hunspell
StringBuilder sb = new StringBuilder();
- FileInfo unsorted = FileSupport.CreateTempFile("unsorted", "dat", tempDir);
- using (OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(unsorted))
+ using FileStream unsorted = FileSupport.CreateTempFileAsStream("unsorted", "dat", tempDir);
+ using (OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(unsorted, leaveOpen: true))
{
foreach (Stream dictionary in dictionaries)
{
@@ -983,7 +969,10 @@ namespace Lucene.Net.Analysis.Hunspell
}
}
- FileInfo sorted = FileSupport.CreateTempFile("sorted", "dat", tempDir);
+ // LUCENENET: Reset the position to the beginning of the stream so we don't have to reopen the file
+ unsorted.Position = 0;
+
+ using FileStream sorted = FileSupport.CreateTempFileAsStream("sorted", "dat", tempDir);
OfflineSorter sorter = new OfflineSorter(Comparer<BytesRef>.Create((o1, o2) =>
{
@@ -1028,14 +1017,7 @@ namespace Lucene.Net.Analysis.Hunspell
}
}));
sorter.Sort(unsorted, sorted);
- try
- {
- unsorted.Delete();
- }
- catch
- {
- // ignore
- }
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
using (OfflineSorter.ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(sorted))
{
@@ -1138,14 +1120,7 @@ namespace Lucene.Net.Analysis.Hunspell
Lucene.Net.Util.Fst.Util.ToUTF32(currentEntry, scratchInts);
words.Add(scratchInts, currentOrds);
}
- try
- {
- sorted.Delete();
- }
- catch
- {
- // ignore
- }
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
internal static char[] DecodeFlags(BytesRef b)
diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs
index e365533b4..d36076977 100644
--- a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs
+++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs
@@ -9,10 +9,8 @@ using Lucene.Net.Util.Automaton;
using Lucene.Net.Util.Fst;
using System;
using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using JCG = J2N.Collections.Generic;
using Int64 = J2N.Numerics.Int64;
+using JCG = J2N.Collections.Generic;
namespace Lucene.Net.Search.Suggest.Analyzing
{
@@ -405,9 +403,10 @@ namespace Lucene.Net.Search.Suggest.Analyzing
throw new ArgumentException("this suggester doesn't support contexts");
}
string prefix = this.GetType().Name;
- var directory = OfflineSorter.GetDefaultTempDir();
- var tempInput = FileSupport.CreateTempFile(prefix, ".input", directory);
- var tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory);
+ var directory = OfflineSorter.DefaultTempDir;
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ using var tempInput = FileSupport.CreateTempFileAsStream(prefix, ".input", directory);
+ using var tempSorted = FileSupport.CreateTempFileAsStream(prefix, ".sorted", directory);
hasPayloads = enumerator.HasPayloads;
@@ -502,13 +501,14 @@ namespace Lucene.Net.Search.Suggest.Analyzing
}
count++;
}
- writer.Dispose();
+
+ tempInput.Position = 0;
// Sort all input/output pairs (required by FST.Builder):
(new OfflineSorter(new AnalyzingComparer(hasPayloads))).Sort(tempInput, tempSorted);
// Free disk space:
- tempInput.Delete();
+ writer.Dispose(); // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
reader = new OfflineSorter.ByteSequencesReader(tempSorted);
@@ -627,9 +627,7 @@ namespace Lucene.Net.Search.Suggest.Analyzing
{
IOUtils.DisposeWhileHandlingException(reader, writer);
}
-
- tempInput.Delete();
- tempSorted.Delete();
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
}
diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs
index ae45c49b4..d16cb1dbd 100644
--- a/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs
+++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs
@@ -270,13 +270,13 @@ namespace Lucene.Net.Search.Suggest.Analyzing
}
string prefix = this.GetType().Name;
- var directory = OfflineSorter.GetDefaultTempDir();
+ var directory = OfflineSorter.DefaultTempDir;
// LUCENENET specific - using GetRandomFileName() instead of picking a random int
DirectoryInfo tempIndexPath; // LUCENENET: IDE0059: Remove unnecessary value assignment
while (true)
{
- tempIndexPath = new DirectoryInfo(Path.Combine(directory.FullName, prefix + ".index." + Path.GetFileNameWithoutExtension(Path.GetRandomFileName())));
+ tempIndexPath = new DirectoryInfo(Path.Combine(directory, prefix + ".index." + Path.GetFileNameWithoutExtension(Path.GetRandomFileName())));
tempIndexPath.Create();
if (System.IO.Directory.Exists(tempIndexPath.FullName))
{
diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs b/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs
index 4c6ae6559..6d842f13c 100644
--- a/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs
+++ b/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs
@@ -1,7 +1,9 @@
using Lucene.Net.Support.IO;
using Lucene.Net.Util;
using System;
+using System.Collections.Generic;
using System.IO;
+using FileStreamOptions = Lucene.Net.Support.IO.FileStreamOptions;
namespace Lucene.Net.Search.Suggest.Fst
{
@@ -22,8 +24,6 @@ namespace Lucene.Net.Search.Suggest.Fst
* limitations under the License.
*/
- using System.Collections.Generic;
-
/// <summary>
/// Builds and iterates over sequences stored on disk.
/// </summary>
@@ -31,8 +31,10 @@ namespace Lucene.Net.Search.Suggest.Fst
{
private readonly OfflineSorter sort;
private OfflineSorter.ByteSequencesWriter writer;
- private FileInfo input;
- private FileInfo sorted;
+ private FileStream input;
+ // LUCENENET specific - removed sorted and made it a local variable of GetEnumerator()
+ private string sortedFileName; // LUCENENET specific
+ private bool isSorted; // LUCENENET specific
/// <summary>
/// Will buffer all sequences to a temporary file and then sort (all on-disk).
@@ -40,8 +42,8 @@ namespace Lucene.Net.Search.Suggest.Fst
public ExternalRefSorter(OfflineSorter sort)
{
this.sort = sort;
- this.input = FileSupport.CreateTempFile("RefSorter-", ".raw", OfflineSorter.GetDefaultTempDir());
- this.writer = new OfflineSorter.ByteSequencesWriter(input);
+ this.input = FileSupport.CreateTempFileAsStream("RefSorter-", ".raw", OfflineSorter.DefaultTempDir);
+ this.writer = new OfflineSorter.ByteSequencesWriter(input, leaveOpen: true);
}
public virtual void Add(BytesRef utf8)
@@ -55,20 +57,29 @@ namespace Lucene.Net.Search.Suggest.Fst
public virtual IBytesRefEnumerator GetEnumerator()
{
- if (sorted is null)
+ if (!isSorted)
{
CloseWriter();
+ input.Position = 0;
- sorted = FileSupport.CreateTempFile("RefSorter-", ".sorted", OfflineSorter.GetDefaultTempDir());
+ using var sorted = FileSupport.CreateTempFileAsStream("RefSorter-", ".sorted", OfflineSorter.DefaultTempDir, EnumeratorFileStreamOptions);
+ sortedFileName = sorted.Name; // LUCENENET specific - store the name so all future calls to GetEnumerator() can open the file.
sort.Sort(input, sorted);
+ isSorted = true; // LUCENENET switched to using a boolean to track whether or not we are sorted so we can dispose sorted in this method.
- input.Delete();
+ input.Dispose(); // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
input = null;
}
- return new ByteSequenceEnumerator(new OfflineSorter.ByteSequencesReader(sorted), sort.Comparer);
+ var sortedClone = new FileStream(sortedFileName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete, bufferSize: OfflineSorter.DEFAULT_FILESTREAM_BUFFER_SIZE);
+ return new ByteSequenceEnumerator(new OfflineSorter.ByteSequencesReader(sortedClone), sort.Comparer);
}
+ /// <summary>
+ /// LUCENENET specific - permissive file options so we can delete the file without errors. We only do that when someone calls <see cref="Dispose()"/> on this class.
+ /// </summary>
+ private static readonly FileStreamOptions EnumeratorFileStreamOptions = new FileStreamOptions { Access = FileAccess.ReadWrite, Share = FileShare.ReadWrite | FileShare.Delete, BufferSize = OfflineSorter.DEFAULT_FILESTREAM_BUFFER_SIZE };
+
private void CloseWriter()
{
if (writer != null)
@@ -99,14 +110,8 @@ namespace Lucene.Net.Search.Suggest.Fst
}
finally
{
- if (input != null)
- {
- input.Delete();
- }
- if (sorted != null)
- {
- sorted.Delete();
- }
+ input?.Dispose();
+ File.Delete(sortedFileName);
}
}
}
diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs b/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs
index 49375271c..0d48fdc9c 100644
--- a/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs
+++ b/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs
@@ -153,8 +153,8 @@ namespace Lucene.Net.Search.Suggest.Fst
{
throw new ArgumentException("this suggester doesn't support contexts");
}
- FileInfo tempInput = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".input", OfflineSorter.GetDefaultTempDir());
- FileInfo tempSorted = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".sorted", OfflineSorter.GetDefaultTempDir());
+ using FileStream tempInput = FileSupport.CreateTempFileAsStream(typeof(FSTCompletionLookup).Name, ".input", OfflineSorter.DefaultTempDir);
+ using FileStream tempSorted = FileSupport.CreateTempFileAsStream(typeof(FSTCompletionLookup).Name, ".sorted", OfflineSorter.DefaultTempDir);
OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
OfflineSorter.ByteSequencesReader reader = null;
@@ -182,12 +182,13 @@ namespace Lucene.Net.Search.Suggest.Fst
output.WriteBytes(spare.Bytes, spare.Offset, spare.Length);
writer.Write(buffer, 0, output.Position);
}
- writer.Dispose();
+ tempInput.Position = 0;
// We don't know the distribution of scores and we need to bucket them, so we'll sort
// and divide into equal buckets.
OfflineSorter.SortInfo info = (new OfflineSorter()).Sort(tempInput, tempSorted);
- tempInput.Delete();
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ writer.Dispose();
FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);
int inputLines = info.Lines;
@@ -241,9 +242,7 @@ namespace Lucene.Net.Search.Suggest.Fst
{
IOUtils.DisposeWhileHandlingException(reader, writer, sorter);
}
-
- tempInput.Delete();
- tempSorted.Delete();
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
}
diff --git a/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
index 584345998..74002753e 100644
--- a/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
+++ b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
@@ -33,8 +33,7 @@ namespace Lucene.Net.Search.Suggest
public class SortedInputEnumerator : IInputEnumerator
{
private readonly IInputEnumerator source;
- private FileInfo tempInput;
- private FileInfo tempSorted;
+ // LUCENENET specific - since these tempInput and tempSorted are only used in the Sort() method, they were moved there.
private readonly OfflineSorter.ByteSequencesReader reader;
private readonly IComparer<BytesRef> comparer;
private readonly bool hasPayloads;
@@ -183,9 +182,10 @@ namespace Lucene.Net.Search.Suggest
private OfflineSorter.ByteSequencesReader Sort()
{
string prefix = this.GetType().Name;
- DirectoryInfo directory = OfflineSorter.GetDefaultTempDir();
- tempInput = FileSupport.CreateTempFile(prefix, ".input", directory);
- tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory);
+ var directory = OfflineSorter.DefaultTempDir;
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ FileStream tempInput = FileSupport.CreateTempFileAsStream(prefix, ".input", directory);
+ FileStream tempSorted = FileSupport.CreateTempFileAsStream(prefix, ".sorted", directory);
var writer = new OfflineSorter.ByteSequencesWriter(tempInput);
bool success = false;
@@ -198,8 +198,11 @@ namespace Lucene.Net.Search.Suggest
{
Encode(writer, output, buffer, source.Current, source.Payload, source.Contexts, source.Weight);
}
- writer.Dispose();
+ tempInput.Position = 0;
(new OfflineSorter(tieBreakByCostComparer)).Sort(tempInput, tempSorted);
+
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ writer.Dispose();
var reader = new OfflineSorter.ByteSequencesReader(tempSorted);
success = true;
return reader;
@@ -214,7 +217,7 @@ namespace Lucene.Net.Search.Suggest
{
try
{
- IOUtils.DisposeWhileHandlingException(writer);
+ IOUtils.DisposeWhileHandlingException(writer, tempSorted);
}
finally
{
@@ -227,14 +230,7 @@ namespace Lucene.Net.Search.Suggest
private void Close()
{
IOUtils.Dispose(reader);
- if (tempInput != null)
- {
- tempInput.Delete();
- }
- if (tempSorted != null)
- {
- tempSorted.Delete();
- }
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
/// <summary>
diff --git a/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
index 7a38535f4..3efb0ad06 100644
--- a/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
+++ b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
@@ -34,8 +34,7 @@ namespace Lucene.Net.Search.Suggest
public class SortedTermFreqEnumeratorWrapper : ITermFreqEnumerator
{
private readonly ITermFreqEnumerator source;
- private FileInfo tempInput;
- private FileInfo tempSorted;
+ // LUCENENET specific - since these tempInput and tempSorted are only used in the Sort() method, they were moved there.
private readonly OfflineSorter.ByteSequencesReader reader;
private readonly IComparer<BytesRef> comparer;
private bool done = false;
@@ -128,9 +127,9 @@ namespace Lucene.Net.Search.Suggest
private OfflineSorter.ByteSequencesReader Sort()
{
string prefix = this.GetType().Name;
- DirectoryInfo directory = OfflineSorter.GetDefaultTempDir();
- tempInput = FileSupport.CreateTempFile(prefix, ".input", directory);
- tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory);
+ var directory = OfflineSorter.DefaultTempDir;
+ FileStream tempInput = FileSupport.CreateTempFileAsStream(prefix, ".input", directory);
+ FileStream tempSorted = FileSupport.CreateTempFileAsStream(prefix, ".sorted", directory);
var writer = new OfflineSorter.ByteSequencesWriter(tempInput);
bool success = false;
@@ -143,8 +142,10 @@ namespace Lucene.Net.Search.Suggest
{
Encode(writer, output, buffer, source.Current, source.Weight);
}
- writer.Dispose();
+ // LUCENENET: Reset the position to the beginning of the stream so we don't have to reopen the file
+ tempInput.Position = 0;
(new OfflineSorter(tieBreakByCostComparer)).Sort(tempInput, tempSorted);
+ writer.Dispose(); // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
OfflineSorter.ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(tempSorted);
success = true;
return reader;
@@ -160,7 +161,7 @@ namespace Lucene.Net.Search.Suggest
{
try
{
- IOUtils.DisposeWhileHandlingException(writer);
+ IOUtils.DisposeWhileHandlingException(writer, tempSorted);
}
finally
{
@@ -173,14 +174,7 @@ namespace Lucene.Net.Search.Suggest
private void Close()
{
IOUtils.Dispose(reader);
- if (tempInput != null)
- {
- tempInput.Delete();
- }
- if (tempSorted != null)
- {
- tempSorted.Delete();
- }
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
/// <summary>
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Hunspell/Test64kAffixes.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Hunspell/Test64kAffixes.cs
index 700f0d2c2..e210ee9fd 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Hunspell/Test64kAffixes.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Hunspell/Test64kAffixes.cs
@@ -55,8 +55,8 @@ namespace Lucene.Net.Analysis.Hunspell
dictWriter.Write("1\ndrink/2\n");
dictWriter.Dispose();
- using Stream affStream = new FileStream(affix.FullName, FileMode.OpenOrCreate);
- using Stream dictStream = new FileStream(dict.FullName, FileMode.OpenOrCreate);
+ using Stream affStream = new FileStream(affix.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
+ using Stream dictStream = new FileStream(dict.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
Dictionary dictionary = new Dictionary(affStream, dictStream);
Stemmer stemmer = new Stemmer(dictionary);
diff --git a/src/Lucene.Net.Tests.Suggest/Support/TestApiConsistency.cs b/src/Lucene.Net.Tests.Suggest/Support/TestApiConsistency.cs
index 87b0bcdf2..cee37fa48 100644
--- a/src/Lucene.Net.Tests.Suggest/Support/TestApiConsistency.cs
+++ b/src/Lucene.Net.Tests.Suggest/Support/TestApiConsistency.cs
@@ -38,7 +38,7 @@ namespace Lucene.Net.Tests.Suggest
[TestCase(typeof(Lucene.Net.Search.Suggest.IInputEnumerator))]
public override void TestPrivateFieldNames(Type typeFromTargetAssembly)
{
- base.TestPrivateFieldNames(typeFromTargetAssembly, @"Snowball\.Ext\..+Stemmer");
+ base.TestPrivateFieldNames(typeFromTargetAssembly, @"Snowball\.Ext\..+Stemmer|EnumeratorFileStreamOptions");
}
[Test, LuceneNetSpecific]
diff --git a/src/Lucene.Net.Tests/Util/TestOfflineSorter.cs b/src/Lucene.Net.Tests/Util/TestOfflineSorter.cs
index 93e1b57f4..55fb91152 100644
--- a/src/Lucene.Net.Tests/Util/TestOfflineSorter.cs
+++ b/src/Lucene.Net.Tests/Util/TestOfflineSorter.cs
@@ -84,7 +84,7 @@ namespace Lucene.Net.Util
public virtual void TestIntermediateMerges()
{
// Sort 20 mb worth of data with 1mb buffer, binary merging.
- OfflineSorter.SortInfo info = CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(1), OfflineSorter.GetDefaultTempDir(), 2), GenerateRandom((int)OfflineSorter.MB * 20));
+ OfflineSorter.SortInfo info = CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(1), OfflineSorter.DefaultTempDir, 2), GenerateRandom((int)OfflineSorter.MB * 20));
Assert.IsTrue(info.MergeRounds > 10);
}
@@ -92,7 +92,7 @@ namespace Lucene.Net.Util
public virtual void TestSmallRandom()
{
// Sort 20 mb worth of data with 1mb buffer.
- OfflineSorter.SortInfo sortInfo = CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(1), OfflineSorter.GetDefaultTempDir(), OfflineSorter.MAX_TEMPFILES), GenerateRandom((int)OfflineSorter.MB * 20));
+ OfflineSorter.SortInfo sortInfo = CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(1), OfflineSorter.DefaultTempDir, OfflineSorter.MAX_TEMPFILES), GenerateRandom((int)OfflineSorter.MB * 20));
Assert.AreEqual(1, sortInfo.MergeRounds);
}
@@ -101,7 +101,7 @@ namespace Lucene.Net.Util
public virtual void TestLargerRandom()
{
// Sort 100MB worth of data with 15mb buffer.
- CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(16), OfflineSorter.GetDefaultTempDir(), OfflineSorter.MAX_TEMPFILES), GenerateRandom((int)OfflineSorter.MB * 100));
+ CheckSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARER, OfflineSorter.BufferSize.Megabytes(16), OfflineSorter.DefaultTempDir, OfflineSorter.MAX_TEMPFILES), GenerateRandom((int)OfflineSorter.MB * 100));
}
private byte[][] GenerateRandom(int howMuchData)
@@ -136,12 +136,13 @@ namespace Lucene.Net.Util
/// </summary>
private OfflineSorter.SortInfo CheckSort(OfflineSorter sort, byte[][] data)
{
- FileInfo unsorted = WriteAll("unsorted", data);
+ using FileStream unsorted = WriteAll("unsorted", data);
Array.Sort(data, unsignedByteOrderComparer);
- FileInfo golden = WriteAll("golden", data);
+ using FileStream golden = WriteAll("golden", data);
- FileInfo sorted = new FileInfo(Path.Combine(tempDir.FullName, "sorted"));
+ string sortedFile = Path.Combine(tempDir.FullName, "sorted");
+ using FileStream sorted = new FileStream(sortedFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, bufferSize: OfflineSorter.DEFAULT_FILESTREAM_BUFFER_SIZE, FileOptions.DeleteOnClose);
OfflineSorter.SortInfo sortInfo = sort.Sort(unsorted, sorted);
//System.out.println("Input size [MB]: " + unsorted.Length() / (1024 * 1024));
//System.out.println(sortInfo);
@@ -153,15 +154,16 @@ namespace Lucene.Net.Util
/// <summary>
/// Make sure two files are byte-byte identical.
/// </summary>
- private void AssertFilesIdentical(FileInfo golden, FileInfo sorted)
+ // LUCENENET specific - switched to using FileStream rather than FileInfo
+ private void AssertFilesIdentical(FileStream golden, FileStream sorted)
{
Assert.AreEqual(golden.Length, sorted.Length);
byte[] buf1 = new byte[64 * 1024];
byte[] buf2 = new byte[64 * 1024];
int len;
- using Stream is1 = golden.Open(FileMode.Open, FileAccess.Read, FileShare.Delete);
- using Stream is2 = sorted.Open(FileMode.Open, FileAccess.Read, FileShare.Delete);
+ Stream is1 = golden;
+ Stream is2 = sorted;
while ((len = is1.Read(buf1, 0, buf1.Length)) > 0)
{
is2.Read(buf2, 0, len);
@@ -172,17 +174,19 @@ namespace Lucene.Net.Util
}
}
- private FileInfo WriteAll(string name, byte[][] data)
+ // LUCENENET specific - switched to using FileStream rather than FileInfo
+ private FileStream WriteAll(string name, byte[][] data)
{
FileInfo file = new FileInfo(Path.Combine(tempDir.FullName, name));
- using (file.Create()) { }
- OfflineSorter.ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(file);
+ var stream = new FileStream(file.FullName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, bufferSize: OfflineSorter.DEFAULT_FILESTREAM_BUFFER_SIZE, FileOptions.DeleteOnClose);
+ OfflineSorter.ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(stream, leaveOpen: true);
foreach (byte[] datum in data)
{
w.Write(datum);
}
w.Dispose();
- return file;
+ stream.Position = 0; // LUCENENET specific - reset the position back to the start of the file so we don't need to reopen it.
+ return stream;
}
[Test]
diff --git a/src/Lucene.Net/Support/IO/FileSupport.cs b/src/Lucene.Net/Support/IO/FileSupport.cs
index 2cf118811..200c53df2 100644
--- a/src/Lucene.Net/Support/IO/FileSupport.cs
+++ b/src/Lucene.Net/Support/IO/FileSupport.cs
@@ -88,7 +88,7 @@ namespace Lucene.Net.Support.IO
{
File.Delete(fileName);
}
- catch { }
+ catch { /* ignored */ }
}
return null; // Should never get here
}
@@ -224,6 +224,92 @@ namespace Lucene.Net.Support.IO
return new FileInfo(stream.Name);
}
+ /// <summary>
+ /// Creates a new empty file in the specified directory, using the given prefix and suffix strings to generate its name and returns an open stream to it.
+ /// </summary>
+ /// <remarks>
+ /// If this method returns successfully then it is guaranteed that:
+ /// <list type="number">
+ /// <item><description>The file denoted by the returned abstract pathname did not exist before this method was invoked, and</description></item>
+ /// <item><description>Neither this method nor any of its variants will return the same abstract pathname again in the current invocation of the application.</description></item>
+ /// </list>
+ /// This method provides only part of a temporary-file facility. However, the file will not be deleted automatically,
+ /// it must be deleted by the caller.
+ /// <para/>
+ /// The prefix argument must be at least three characters long. It is recommended that the prefix be a short, meaningful
+ /// string such as "hjb" or "mail".
+ /// <para/>
+ /// The suffix argument may be null, in which case a random suffix will be used.
+ /// <para/>
+ /// Both prefix and suffix must be provided with valid characters for the underlying system, as specified by
+ /// <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// <para/>
+ /// If the directory argument is null then the system-dependent default temporary-file directory will be used,
+ /// with a random subdirectory name. The default temporary-file directory is specified by the
+ /// <see cref="Path.GetTempPath()"/> method. On UNIX systems the default value of this property is typically
+ /// "/tmp" or "/var/tmp"; on Microsoft Windows systems it is typically "C:\\Users\\[UserName]\\AppData\Local\Temp".
+ /// </remarks>
+ /// <param name="prefix">The prefix string to be used in generating the file's name; must be at least three characters long</param>
+ /// <param name="suffix">The suffix string to be used in generating the file's name; may be null, in which case a random suffix will be generated</param>
+ /// <param name="directory">The directory in which the file is to be created, or null if the default temporary-file directory is to be used</param>
+ /// <returns>A <see cref="FileStream"/> instance representing the temp file that was created.</returns>
+ /// <exception cref="ArgumentNullException"><paramref name="prefix"/> is <c>null</c>.</exception>
+ /// <exception cref="ArgumentException">
+ /// <paramref name="prefix"/> length is less than 3 characters.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="prefix"/> or <paramref name="suffix"/> contains invalid characters according to <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// </exception>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static FileStream CreateTempFileAsStream(string prefix, string? suffix, DirectoryInfo? directory)
+ {
+ return CreateTempFileAsStream(prefix, suffix, directory?.FullName, DefaultFileStreamOptions);
+ }
+
+ /// <summary>
+ /// Creates a new empty file in the specified directory, using the given prefix and suffix strings to generate its name and returns an open stream to it.
+ /// </summary>
+ /// <remarks>
+ /// If this method returns successfully then it is guaranteed that:
+ /// <list type="number">
+ /// <item><description>The file denoted by the returned abstract pathname did not exist before this method was invoked, and</description></item>
+ /// <item><description>Neither this method nor any of its variants will return the same abstract pathname again in the current invocation of the application.</description></item>
+ /// </list>
+ /// This method provides only part of a temporary-file facility. However, the file will not be deleted automatically,
+ /// it must be deleted by the caller.
+ /// <para/>
+ /// The prefix argument must be at least three characters long. It is recommended that the prefix be a short, meaningful
+ /// string such as "hjb" or "mail".
+ /// <para/>
+ /// The suffix argument may be null, in which case a random suffix will be used.
+ /// <para/>
+ /// Both prefix and suffix must be provided with valid characters for the underlying system, as specified by
+ /// <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// <para/>
+ /// If the directory argument is null then the system-dependent default temporary-file directory will be used,
+ /// with a random subdirectory name. The default temporary-file directory is specified by the
+ /// <see cref="Path.GetTempPath()"/> method. On UNIX systems the default value of this property is typically
+ /// "/tmp" or "/var/tmp"; on Microsoft Windows systems it is typically "C:\\Users\\[UserName]\\AppData\Local\Temp".
+ /// </remarks>
+ /// <param name="prefix">The prefix string to be used in generating the file's name; must be at least three characters long</param>
+ /// <param name="suffix">The suffix string to be used in generating the file's name; may be null, in which case a random suffix will be generated</param>
+ /// <param name="directory">The directory in which the file is to be created, or null if the default temporary-file directory is to be used</param>
+ /// <returns>A <see cref="FileStream"/> instance representing the temp file that was created.</returns>
+ /// <exception cref="ArgumentNullException"><paramref name="prefix"/> is <c>null</c>.</exception>
+ /// <exception cref="ArgumentException">
+ /// <paramref name="prefix"/> length is less than 3 characters.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="prefix"/> or <paramref name="suffix"/> contains invalid characters according to <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// </exception>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static FileStream CreateTempFileAsStream(string prefix, string? suffix, string? directory)
+ {
+ return CreateTempFileAsStream(prefix, suffix, directory, DefaultFileStreamOptions);
+ }
+
/// <summary>
/// Creates a new empty file in the specified directory, using the given prefix and suffix strings to generate its name and returns an open stream to it.
/// </summary>
@@ -261,6 +347,10 @@ namespace Lucene.Net.Support.IO
/// -or-
/// <para/>
/// <paramref name="prefix"/> or <paramref name="suffix"/> contains invalid characters according to <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="options"/>.<see cref="FileStreamOptions.Access"/> is set to <see cref="FileAccess.Read"/>.
/// </exception>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static FileStream CreateTempFileAsStream(string prefix, string? suffix, DirectoryInfo? directory, FileStreamOptions options)
@@ -305,6 +395,10 @@ namespace Lucene.Net.Support.IO
/// -or-
/// <para/>
/// <paramref name="prefix"/> or <paramref name="suffix"/> contains invalid characters according to <see cref="Path.GetInvalidFileNameChars()"/>.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="options"/>.<see cref="FileStreamOptions.Access"/> is set to <see cref="FileAccess.Read"/>.
/// </exception>
public static FileStream CreateTempFileAsStream(string prefix, string? suffix, string? directory, FileStreamOptions options)
{
@@ -312,12 +406,17 @@ namespace Lucene.Net.Support.IO
throw new ArgumentNullException(nameof(prefix));
if (prefix.Length < 3)
throw new ArgumentException("Prefix string too short");
+ if (options is null)
+ throw new ArgumentNullException(nameof(options));
// Ensure the strings passed don't contain invalid characters
if (prefix.ContainsAny(INVALID_FILENAME_CHARS))
throw new ArgumentException(string.Format("Prefix contains invalid characters. You may not use any of '{0}'", string.Join(", ", INVALID_FILENAME_CHARS)));
if (suffix != null && suffix.ContainsAny(INVALID_FILENAME_CHARS))
throw new ArgumentException(string.Format("Suffix contains invalid characters. You may not use any of '{0}'", string.Join(", ", INVALID_FILENAME_CHARS)));
+ if (options.Access == FileAccess.Read)
+ throw new ArgumentException("Read-only for options.FileAccess is not supported.");
+
// If no directory supplied, create one.
if (directory is null)
diff --git a/src/Lucene.Net/Util/OfflineSorter.cs b/src/Lucene.Net/Util/OfflineSorter.cs
index afd208770..d484aa1dc 100644
--- a/src/Lucene.Net/Util/OfflineSorter.cs
+++ b/src/Lucene.Net/Util/OfflineSorter.cs
@@ -40,6 +40,18 @@ namespace Lucene.Net.Util
/// </summary>
public sealed class OfflineSorter
{
+ /// <summary>
+ /// The default encoding (UTF-8 without a byte order mark) used by <see cref="ByteSequencesReader"/> and <see cref="ByteSequencesWriter"/>.
+ /// This encoding should always be used when calling the constructor overloads that accept <see cref="BinaryReader"/> or <see cref="BinaryWriter"/>.
+ /// </summary>
+ public static readonly Encoding DEFAULT_ENCODING = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
+
+ /// <summary>
+ /// The recommended buffer size to use on <see cref="Sort(FileStream, FileStream)"/> or when creating a
+ /// <see cref="ByteSequencesReader"/> and <see cref="ByteSequencesWriter"/>.
+ /// </summary>
+ public const int DEFAULT_FILESTREAM_BUFFER_SIZE = 8192;
+
/// <summary>
/// Convenience constant for megabytes </summary>
public const long MB = 1024 * 1024;
@@ -192,7 +204,7 @@ namespace Lucene.Net.Util
}
private readonly BufferSize ramBufferSize;
- private readonly DirectoryInfo tempDirectory;
+ private readonly string tempDirectory;
private readonly Counter bufferBytesUsed = Counter.NewCounter();
private readonly BytesRefArray buffer;
@@ -204,23 +216,28 @@ namespace Lucene.Net.Util
/// Default comparer: sorts in binary (codepoint) order </summary>
public static readonly IComparer<BytesRef> DEFAULT_COMPARER = Utf8SortedAsUnicodeComparer.Instance;
+ /// <summary>
+ /// LUCENENET specific - cache the temp directory path so we can return it from a property.
+ /// </summary>
+ private static readonly string DEFAULT_TEMP_DIR = Path.GetTempPath();
+
/// <summary>
/// Defaults constructor.
/// </summary>
- /// <seealso cref="GetDefaultTempDir()"/>
+ /// <seealso cref="DefaultTempDir"/>
/// <seealso cref="BufferSize.Automatic()"/>
public OfflineSorter()
- : this(DEFAULT_COMPARER, BufferSize.Automatic(), GetDefaultTempDir(), MAX_TEMPFILES)
+ : this(DEFAULT_COMPARER, BufferSize.Automatic(), DefaultTempDir, MAX_TEMPFILES)
{
}
/// <summary>
/// Defaults constructor with a custom comparer.
/// </summary>
- /// <seealso cref="GetDefaultTempDir()"/>
+ /// <seealso cref="DefaultTempDir"/>
/// <seealso cref="BufferSize.Automatic()"/>
public OfflineSorter(IComparer<BytesRef> comparer)
- : this(comparer, BufferSize.Automatic(), GetDefaultTempDir(), MAX_TEMPFILES)
+ : this(comparer, BufferSize.Automatic(), DefaultTempDir, MAX_TEMPFILES)
{
}
@@ -231,13 +248,26 @@ namespace Lucene.Net.Util
/// <exception cref="ArgumentException"><paramref name="ramBufferSize"/> bytes are less than <see cref="ABSOLUTE_MIN_SORT_BUFFER_SIZE"/>.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="maxTempfiles"/> is less than 2.</exception>
public OfflineSorter(IComparer<BytesRef> comparer, BufferSize ramBufferSize, DirectoryInfo tempDirectory, int maxTempfiles)
+ : this(comparer, ramBufferSize, tempDirectory?.FullName ?? throw new ArgumentNullException(nameof(tempDirectory)), maxTempfiles)
+ {
+
+ }
+
+ /// <summary>
+ /// All-details constructor.
+ /// </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="comparer"/>, <paramref name="ramBufferSize"/> or <paramref name="tempDirectoryPath"/> is <c>null</c>.</exception>
+ /// <exception cref="ArgumentException"><paramref name="ramBufferSize"/> bytes are less than <see cref="ABSOLUTE_MIN_SORT_BUFFER_SIZE"/>.</exception>
+ /// <exception cref="ArgumentOutOfRangeException"><paramref name="maxTempfiles"/> is less than 2.</exception>
+ // LUCENENET specific
+ public OfflineSorter(IComparer<BytesRef> comparer, BufferSize ramBufferSize, string tempDirectoryPath, int maxTempfiles)
{
if (comparer is null)
throw new ArgumentNullException(nameof(comparer)); // LUCENENET: Added guard clauses
if (ramBufferSize is null)
throw new ArgumentNullException(nameof(ramBufferSize));
- if (tempDirectory is null)
- throw new ArgumentNullException(nameof(tempDirectory));
+ if (tempDirectoryPath is null)
+ throw new ArgumentNullException(nameof(tempDirectoryPath));
buffer = new BytesRefArray(bufferBytesUsed);
if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE)
@@ -251,44 +281,49 @@ namespace Lucene.Net.Util
}
this.ramBufferSize = ramBufferSize;
- this.tempDirectory = tempDirectory;
+ this.tempDirectory = tempDirectoryPath;
this.maxTempFiles = maxTempfiles;
this.comparer = comparer;
}
- /// <summary>
- /// All-details constructor, specifying <paramref name="tempDirectoryPath"/> as a <see cref="string"/>.
- /// </summary>
- /// <exception cref="ArgumentNullException"><paramref name="comparer"/>, <paramref name="ramBufferSize"/> or <paramref name="tempDirectoryPath"/> is <c>null</c>.</exception>
- /// <exception cref="ArgumentException"><paramref name="ramBufferSize"/> bytes are less than <see cref="ABSOLUTE_MIN_SORT_BUFFER_SIZE"/>.</exception>
- /// <exception cref="ArgumentOutOfRangeException"><paramref name="maxTempfiles"/> is less than 2.</exception>
- // LUCENENET specific
- public OfflineSorter(IComparer<BytesRef> comparer, BufferSize ramBufferSize, string tempDirectoryPath, int maxTempfiles)
- : this(comparer, ramBufferSize, string.IsNullOrWhiteSpace(tempDirectoryPath) ? new DirectoryInfo(tempDirectoryPath) : throw new ArgumentException($"{nameof(tempDirectoryPath)} must not be null or empty."), maxTempfiles)
- {
- }
-
/// <summary>
/// Sort input to output, explicit hint for the buffer size. The amount of allocated
/// memory may deviate from the hint (may be smaller or larger).
/// </summary>
+ /// <param name="input">The input stream. Must be both seekable and readable.</param>
+ /// <param name="output">The output stream. Must be seekable and writable.</param>
/// <exception cref="ArgumentNullException"><paramref name="input"/> or <paramref name="output"/> is <c>null</c>.</exception>
- public SortInfo Sort(FileInfo input, FileInfo output)
+ /// <exception cref="ArgumentException">
+ /// <paramref name="input"/> or <paramref name="output"/> is not seekable.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="input"/> is not readable.
+ /// <para/>
+ /// -or-
+ /// <para/>
+ /// <paramref name="output"/> is not writable.
+ /// </exception>
+ /// <exception cref="ArgumentException"><paramref name="input"/> or <paramref name="output"/> is not seekable.</exception>
+ public SortInfo Sort(FileStream input, FileStream output)
{
if (input is null)
throw new ArgumentNullException(nameof(input)); // LUCENENET: Added guard clauses
if (output is null)
throw new ArgumentNullException(nameof(output));
+ if (!input.CanSeek)
+ throw new ArgumentException($"{nameof(input)} stream must be seekable.");
+ if (!output.CanSeek)
+ throw new ArgumentException($"{nameof(output)} stream must be seekable.");
sortInfo = new SortInfo(this) { TotalTime = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond }; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
- output.Delete();
+ // LUCENENET specific - the output is an open stream. We know we don't have to delete it before we start.
- var merges = new JCG.List<FileInfo>();
- bool success2 = false;
+ var merges = new JCG.List<FileStream>();
try
{
- var inputStream = new ByteSequencesReader(input);
+ var inputStream = new ByteSequencesReader(input, leaveOpen: true);
bool success = false;
try
{
@@ -302,18 +337,17 @@ namespace Lucene.Net.Util
// Handle intermediate merges.
if (merges.Count == maxTempFiles)
{
- var intermediate = FileSupport.CreateTempFile("sort", "intermediate", tempDirectory);
+ var intermediate = FileSupport.CreateTempFileAsStream("sort", "intermediate", tempDirectory);
try
{
MergePartitions(merges, intermediate);
}
finally
{
- foreach (var file in merges)
- {
- file.Delete();
- }
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ IOUtils.Dispose(merges);
merges.Clear();
+ intermediate.Position = 0;
merges.Add(intermediate);
}
sortInfo.TempMergeFiles++;
@@ -336,34 +370,23 @@ namespace Lucene.Net.Util
// One partition, try to rename or copy if unsuccessful.
if (merges.Count == 1)
{
- FileInfo single = merges[0];
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
+ using FileStream single = merges[0];
Copy(single, output);
- try
- {
- File.Delete(single.FullName);
- }
- catch
- {
- // ignored
- }
}
else
{
// otherwise merge the partitions with a priority queue.
MergePartitions(merges, output);
}
- success2 = true;
}
finally
{
- foreach (FileInfo file in merges)
- {
- file.Delete();
- }
- if (!success2)
- {
- output.Delete();
- }
+ // LUCENENET: Reset the position to the beginning of the streams so we don't have to reopen the files
+ input.Position = 0;
+ output.Position = 0;
+ IOUtils.Dispose(merges);
+ // LUCENENET specific - we are using the FileOptions.DeleteOnClose FileStream option to delete the file when it is disposed.
}
sortInfo.TotalTime = ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - sortInfo.TotalTime); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
@@ -371,47 +394,49 @@ namespace Lucene.Net.Util
}
/// <summary>
- /// Returns the default temporary directory. By default, the System's temp folder. If not accessible
- /// or not available, an <see cref="IOException"/> is thrown.
+ /// Sort input to output, explicit hint for the buffer size. The amount of allocated
+ /// memory may deviate from the hint (may be smaller or larger).
/// </summary>
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static DirectoryInfo GetDefaultTempDir()
+ /// <exception cref="ArgumentNullException"><paramref name="input"/> or <paramref name="output"/> is <c>null</c>.</exception>
+ public SortInfo Sort(FileInfo input, FileInfo output)
{
- return new DirectoryInfo(Path.GetTempPath());
+ if (input is null)
+ throw new ArgumentNullException(nameof(input)); // LUCENENET: Added guard clauses
+ if (output is null)
+ throw new ArgumentNullException(nameof(output));
+
+ using FileStream inputStream = new FileStream(input.FullName, FileMode.Open, FileAccess.ReadWrite,
+ FileShare.Read, bufferSize: DEFAULT_FILESTREAM_BUFFER_SIZE, FileOptions.DeleteOnClose | FileOptions.RandomAccess);
+ using FileStream outputStream = new FileStream(output.FullName, FileMode.Open, FileAccess.ReadWrite,
+ FileShare.Read, bufferSize: DEFAULT_FILESTREAM_BUFFER_SIZE, FileOptions.DeleteOnClose | FileOptions.RandomAccess);
+ return Sort(inputStream, outputStream);
}
/// <summary>
- /// Returns the default temporary directory. By default, the System's temp folder. If not accessible
- /// or not available, an <see cref="IOException"/> is thrown.
+ /// Returns the default temporary directory. By default, the System's temp folder.
/// </summary>
- [Obsolete("Use GetDefaultTempDir() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
- public static DirectoryInfo DefaultTempDir()
- {
- return new DirectoryInfo(Path.GetTempPath());
- }
+ public static string DefaultTempDir => DEFAULT_TEMP_DIR;
/// <summary>
/// Copies one file to another.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void Copy(FileInfo file, FileInfo output)
+ private static void Copy(FileStream file, FileStream output)
{
- using Stream inputStream = file.OpenRead();
- using Stream outputStream = output.OpenWrite();
- inputStream.CopyTo(outputStream);
+ file.CopyTo(output);
}
/// <summary>
/// Sort a single partition in-memory. </summary>
- private FileInfo SortPartition(/*int len*/) // LUCENENET NOTE: made private, since protected is not valid in a sealed class. Also eliminated unused parameter.
+ private FileStream SortPartition(/*int len*/) // LUCENENET NOTE: made private, since protected is not valid in a sealed class. Also eliminated unused parameter.
{
var data = this.buffer;
- FileInfo tempFile = FileSupport.CreateTempFile("sort", "partition", tempDirectory);
+ FileStream tempFile = FileSupport.CreateTempFileAsStream("sort", "partition", tempDirectory);
long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
sortInfo!.SortTime += ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
- using (var @out = new ByteSequencesWriter(tempFile))
+ using (var @out = new ByteSequencesWriter(tempFile, leaveOpen: true))
{
IBytesRefEnumerator iter = buffer.GetEnumerator(comparer);
while (iter.MoveNext())
@@ -423,16 +448,17 @@ namespace Lucene.Net.Util
// Clean up the buffer for the next partition.
data.Clear();
+ tempFile.Position = 0;
return tempFile;
}
/// <summary>
/// Merge a list of sorted temporary files (partitions) into an output file. </summary>
- internal void MergePartitions(IList<FileInfo> merges, FileInfo outputFile)
+ internal void MergePartitions(IList<FileStream> merges, FileStream outputFile)
{
long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
- var @out = new ByteSequencesWriter(outputFile);
+ var @out = new ByteSequencesWriter(outputFile, leaveOpen: true);
PriorityQueue<FileAndTop> queue = new PriorityQueueAnonymousClass(this, merges.Count);
@@ -442,7 +468,7 @@ namespace Lucene.Net.Util
// Open streams and read the top for each file
for (int i = 0; i < merges.Count; i++)
{
- streams[i] = new ByteSequencesReader(merges[i]);
+ streams[i] = new ByteSequencesReader(merges[i], leaveOpen: true);
byte[]? line = streams[i].Read();
if (line is not null)
{
@@ -536,73 +562,57 @@ namespace Lucene.Net.Util
}
}
-
/// <summary>
/// Utility class to emit length-prefixed <see cref="T:byte[]"/> entries to an output stream for sorting.
/// Complementary to <see cref="ByteSequencesReader"/>.
/// </summary>
public class ByteSequencesWriter : IDisposable
{
- private readonly DataOutput os;
+ private readonly BinaryWriter os;
private bool disposed; // LUCENENET specific
/// <summary>
- /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="FileInfo"/>. </summary>
- /// <exception cref="ArgumentNullException"><paramref name="file"/> is <c>null</c>.</exception>
- public ByteSequencesWriter(FileInfo file)
- : this(NewBinaryWriterDataOutput(file))
+ /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="FileStream"/>. </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="stream"/> is <c>null</c>.</exception>
+ public ByteSequencesWriter(FileStream stream)
+ : this(new BinaryWriter(stream, DEFAULT_ENCODING, leaveOpen: false))
{
}
/// <summary>
- /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="FileInfo"/>. </summary>
- /// <exception cref="ArgumentException"><paramref name="path"/> is <c>null</c> or whitespace.</exception>
- public ByteSequencesWriter(string path)
- : this(NewBinaryWriterDataOutput(path))
+ /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="FileStream"/>. </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="stream"/> is <c>null</c>.</exception>
+ public ByteSequencesWriter(FileStream stream, bool leaveOpen)
+ : this(new BinaryWriter(stream, DEFAULT_ENCODING, leaveOpen))
{
}
/// <summary>
- /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="DataOutput"/>. </summary>
- /// <exception cref="ArgumentNullException"><paramref name="os"/> is <c>null</c>.</exception>
- public ByteSequencesWriter(DataOutput os)
+ /// Constructs a <see cref="ByteSequencesWriter"/> to the provided file path. </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="path"/> is <c>null</c>.</exception>
+ public ByteSequencesWriter(string path)
+ : this(new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.Read, bufferSize: DEFAULT_FILESTREAM_BUFFER_SIZE))
{
- this.os = os ?? throw new ArgumentNullException(nameof(os)); // LUCENENET: Added guard clause
}
/// <summary>
- /// LUCENENET specific - ensures the file has been created with no BOM
- /// if it doesn't already exist and opens the file for writing.
- /// Java doesn't use a BOM by default.
- /// </summary>
+ /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="FileInfo"/>. </summary>
/// <exception cref="ArgumentNullException"><paramref name="file"/> is <c>null</c>.</exception>
- private static BinaryWriterDataOutput NewBinaryWriterDataOutput(FileInfo file)
+ // LUCENENET specific - This is for bw compatibility with an earlier approach using FileInfo (similar to how it worked in Java)
+ public ByteSequencesWriter(FileInfo file)
+ : this(file?.FullName ?? throw new ArgumentNullException(nameof(file)))
{
- if (file is null)
- throw new ArgumentNullException(nameof(file));
-
- return NewBinaryWriterDataOutput(file.FullName);
}
/// <summary>
- /// LUCENENET specific - ensures the file has been created with no BOM
- /// if it doesn't already exist and opens the file for writing.
- /// Java doesn't use a BOM by default.
+ /// Constructs a <see cref="ByteSequencesWriter"/> to the provided <see cref="BinaryWriter"/>.
+ /// <b>NOTE:</b> To match Lucene, pass the <paramref name="writer"/>'s constructor the
+ /// <see cref="DEFAULT_ENCODING"/>, which is UTF-8 without a byte order mark.
/// </summary>
- /// <exception cref="ArgumentException"><paramref name="path"/> is <c>null</c> or whitespace.</exception>
- private static BinaryWriterDataOutput NewBinaryWriterDataOutput(string path)
+ /// <exception cref="ArgumentNullException"><paramref name="writer"/> is <c>null</c>.</exception>
+ public ByteSequencesWriter(BinaryWriter writer)
{
- if (string.IsNullOrWhiteSpace(path))
- throw new ArgumentException($"{nameof(path)} may not be null or whitespace.");
-
- // Create the file (without BOM) if it doesn't already exist
- if (!File.Exists(path))
- {
- // Create the file
- File.WriteAllText(path, string.Empty, new UTF8Encoding(false) /* No BOM */);
- }
-
- return new BinaryWriterDataOutput(new BinaryWriter(new FileStream(path, FileMode.Open, FileAccess.Write)));
+ this.os = writer ?? throw new ArgumentNullException(nameof(writer)); // LUCENENET: Added guard clause
}
/// <summary>
@@ -648,8 +658,8 @@ namespace Lucene.Net.Util
if (off > bytes.Length - len) // Checks for int overflow
throw new ArgumentException("Index and length must refer to a location within the array.");
- os.WriteInt16((short)len);
- os.WriteBytes(bytes, off, len); // LUCENENET NOTE: We call WriteBytes, since there is no Write() on Lucene's version of DataOutput
+ os.Write((short)len);
+ os.Write(bytes, off, len);
}
/// <summary>
@@ -666,12 +676,11 @@ namespace Lucene.Net.Util
/// </summary>
protected virtual void Dispose(bool disposing) // LUCENENET specific - implemented proper dispose pattern
{
- if (!disposed && disposing && this.os is IDisposable disposable)
+ if (!disposed && disposing)
{
- disposable.Dispose();
+ os.Dispose();
disposed = true;
}
-
}
}
@@ -681,16 +690,22 @@ namespace Lucene.Net.Util
/// </summary>
public class ByteSequencesReader : IDisposable
{
- private readonly DataInput inputStream;
+ private readonly BinaryReader @is;
private bool disposed; // LUCENENET specific
/// <summary>
- /// Constructs a <see cref="ByteSequencesReader"/> from the provided <see cref="FileInfo"/>. </summary>
- /// <exception cref="ArgumentNullException"><paramref name="file"/> is <c>null</c>.</exception>
- public ByteSequencesReader(FileInfo file)
- : this(file is not null ?
- new BinaryReaderDataInput(new BinaryReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read))) :
- throw new ArgumentNullException(nameof(file))) // LUCENENET: Added guard clause
+ /// Constructs a <see cref="ByteSequencesReader"/> from the provided <see cref="FileStream"/>. </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="stream"/> is <c>null</c>.</exception>
+ public ByteSequencesReader(FileStream stream)
+ : this(new BinaryReader(stream, DEFAULT_ENCODING, leaveOpen: false))
+ {
+ }
+
+ /// <summary>
+ /// Constructs a <see cref="ByteSequencesReader"/> from the provided <see cref="FileStream"/>. </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="stream"/> is <c>null</c>.</exception>
+ public ByteSequencesReader(FileStream stream, bool leaveOpen)
+ : this(new BinaryReader(stream, DEFAULT_ENCODING, leaveOpen))
{
}
@@ -699,16 +714,29 @@ namespace Lucene.Net.Util
/// <exception cref="ArgumentException"><paramref name="path"/> is <c>null</c> or whitespace.</exception>
// LUCENENET specific
public ByteSequencesReader(string path)
- : this(!string.IsNullOrWhiteSpace(path) ? new FileInfo(path) : throw new ArgumentException($"{nameof(path)} may not be null or whitespace."))
+ : this(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: DEFAULT_FILESTREAM_BUFFER_SIZE))
+ {
+ }
+
+ /// <summary>
+ /// Constructs a <see cref="ByteSequencesReader"/> from the provided <paramref name="file"/>. </summary>
+ /// <exception cref="ArgumentException"><paramref name="file"/> is <c>null</c> or whitespace.</exception>
+ // LUCENENET specific - This is for bw compatibility with an earlier approach using FileInfo (similar to how it worked in Java)
+ public ByteSequencesReader(FileInfo file)
+ : this(file?.FullName ?? throw new ArgumentNullException(nameof(file)))
{
}
/// <summary>
- /// Constructs a <see cref="ByteSequencesReader"/> from the provided <see cref="DataInput"/>. </summary>
- /// <exception cref="ArgumentNullException"><paramref name="inputStream"/> is <c>null</c>.</exception>
- public ByteSequencesReader(DataInput inputStream)
+ /// Constructs a <see cref="ByteSequencesReader"/> from the provided <see cref="BinaryReader"/>.
+ /// <para/>
+ /// <b>NOTE:</b> To match Lucene, pass the <paramref name="reader"/>'s constructor the
+ /// <see cref="DEFAULT_ENCODING"/>, which is UTF-8 without a byte order mark.
+ /// </summary>
+ /// <exception cref="ArgumentNullException"><paramref name="reader"/> is <c>null</c>.</exception>
+ public ByteSequencesReader(BinaryReader reader)
{
- this.inputStream = inputStream ?? throw new ArgumentNullException(nameof(inputStream)); // LUCENENET: Added guard clause
+ this.@is = reader ?? throw new ArgumentNullException(nameof(reader)); // LUCENENET: Added guard clause
}
/// <summary>
@@ -727,7 +755,7 @@ namespace Lucene.Net.Util
ushort length;
try
{
- length = (ushort)inputStream.ReadInt16();
+ length = (ushort)@is.ReadInt16();
}
catch (Exception e) when (e.IsEOFException())
{
@@ -737,7 +765,7 @@ namespace Lucene.Net.Util
@ref.Grow(length);
@ref.Offset = 0;
@ref.Length = length;
- inputStream.ReadBytes(@ref.Bytes, 0, length);
+ @is.Read(@ref.Bytes, 0, length);
return true;
}
@@ -753,7 +781,7 @@ namespace Lucene.Net.Util
ushort length;
try
{
- length = (ushort)inputStream.ReadInt16();
+ length = (ushort)@is.ReadInt16();
}
catch (Exception e) when (e.IsEOFException())
{
@@ -762,7 +790,7 @@ namespace Lucene.Net.Util
if (Debugging.AssertsEnabled) Debugging.Assert(length >= 0, "Sanity: sequence length < 0: {0}", length);
byte[] result = new byte[length];
- inputStream.ReadBytes(result, 0, length);
+ @is.Read(result, 0, length);
return result;
}
@@ -777,9 +805,9 @@ namespace Lucene.Net.Util
protected virtual void Dispose(bool disposing) // LUCENENET specific - implemented proper dispose pattern
{
- if (!disposed && disposing && this.inputStream is IDisposable disposable)
+ if (!disposed && disposing)
{
- disposable.Dispose();
+ @is.Dispose();
disposed = true;
}
}