You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/09/13 22:01:47 UTC
[1/2] lucenenet git commit: Revert "Lucene.Net.Store.FSDirectory:
Removed Fsync() method and m_staleFiles variable and all references to them"
to prevent widespread 'file in use by another process' error.
Repository: lucenenet
Updated Branches:
refs/heads/master 03bc1cbff -> 8eada7984
Revert "Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles variable and all references to them" to prevent widespread 'file in use by another process' error.
This reverts commit bc295b0e79c153d9dfa7e9f99a144db135a014f5.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/681430e4
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/681430e4
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/681430e4
Branch: refs/heads/master
Commit: 681430e44ce316e18b788598ed002f6f14dd3822
Parents: 03bc1cb
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Thu Sep 14 04:12:33 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Thu Sep 14 04:12:33 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Store/FSDirectory.cs | 55 +++++------
src/Lucene.Net/Util/IOUtils.cs | 153 +++++++++++++++----------------
2 files changed, 96 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Store/FSDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs
index 8c65d9c..5487b96 100644
--- a/src/Lucene.Net/Store/FSDirectory.cs
+++ b/src/Lucene.Net/Store/FSDirectory.cs
@@ -92,10 +92,7 @@ namespace Lucene.Net.Store
public const int DEFAULT_READ_CHUNK_SIZE = 8192;
protected readonly DirectoryInfo m_directory; // The underlying filesystem directory
-
- // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
- // our FileStream is disposed.
- //protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed
+ protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed
#pragma warning disable 612, 618
private int chunkSize = DEFAULT_READ_CHUNK_SIZE;
#pragma warning restore 612, 618
@@ -313,9 +310,7 @@ namespace Lucene.Net.Store
{
throw new IOException("Cannot delete " + file, e);
}
- // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
- // our FileStream is disposed.
- //m_staleFiles.Remove(name);
+ m_staleFiles.Remove(name);
}
/// <summary>
@@ -358,35 +353,30 @@ namespace Lucene.Net.Store
protected virtual void OnIndexOutputClosed(FSIndexOutput io)
{
- // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
- // our FileStream is disposed.
- //m_staleFiles.Add(io.name);
+ m_staleFiles.Add(io.name);
}
public override void Sync(ICollection<string> names)
{
EnsureOpen();
+ ISet<string> toSync = new HashSet<string>(names);
+ toSync.IntersectWith(m_staleFiles);
- // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
- // our FileStream is disposed. Therefore, there is nothing else to do in this method.
- //ISet<string> toSync = new HashSet<string>(names);
- //toSync.IntersectWith(m_staleFiles);
-
- //// LUCENENET specific: Fsync breaks concurrency here.
- //// Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
- ////foreach (var name in toSync)
- ////{
- //// Fsync(name);
- ////}
-
- //// fsync the directory itsself, but only if there was any file fsynced before
- //// (otherwise it can happen that the directory does not yet exist)!
- //if (toSync.Count > 0)
+ // LUCENENET specific: Fsync breaks concurrency here.
+ // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+ //foreach (var name in toSync)
//{
- // IOUtils.Fsync(m_directory.FullName, true);
+ // Fsync(name);
//}
- //m_staleFiles.ExceptWith(toSync);
+ // fsync the directory itsself, but only if there was any file fsynced before
+ // (otherwise it can happen that the directory does not yet exist)!
+ if (toSync.Count > 0)
+ {
+ IOUtils.Fsync(m_directory.FullName, true);
+ }
+
+ m_staleFiles.ExceptWith(toSync);
}
public override string GetLockID()
@@ -545,12 +535,9 @@ namespace Lucene.Net.Store
// LUCENENET NOTE: FileStream doesn't have a way to set length
}
- // LUCENENET specific: Fsync is pointless in .NET, since we are
- // calling FileStream.Flush(true) before the stream is disposed
- // which means we never need it at the point in Java where it is called.
- //protected virtual void Fsync(string name)
- //{
- // IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);
- //}
+ protected virtual void Fsync(string name)
+ {
+ IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Util/IOUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs
index 27daafa..5907b60 100644
--- a/src/Lucene.Net/Util/IOUtils.cs
+++ b/src/Lucene.Net/Util/IOUtils.cs
@@ -522,83 +522,80 @@ namespace Lucene.Net.Util
}
}
- // LUCENENET specific: Fsync is pointless in .NET, since we are
- // calling FileStream.Flush(true) before the stream is disposed
- // which means we never need it at the point in Java where it is called.
- // /// <summary>
- // /// Ensure that any writes to the given file is written to the storage device that contains it. </summary>
- // /// <param name="fileToSync"> The file to fsync </param>
- // /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s,
- // /// because not all file systems and operating systems allow to fsync on a directory) </param>
- // public static void Fsync(string fileToSync, bool isDir)
- // {
- // // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version
- // // they catch this in IOException branch and return if the call is for the directory.
- // // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by
- // // IOException block. No need to even attempt to fsync, just return if the call is for directory
- // if (isDir)
- // {
- // return;
- // }
-
- // var retryCount = 1;
- // while (true)
- // {
- // FileStream file = null;
- // bool success = false;
- // try
- // {
- // // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
- // // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
- // file = new FileStream(fileToSync,
- // FileMode.Open, // We shouldn't create a file when syncing.
- // // Java version uses FileChannel which doesn't create the file if it doesn't already exist,
- // // so there should be no reason for attempting to create it in Lucene.Net.
- // FileAccess.Write,
- // FileShare.ReadWrite);
- // //FileSupport.Sync(file);
- // file.Flush(true);
- // success = true;
- // }
- //#pragma warning disable 168
- // catch (IOException e)
- //#pragma warning restore 168
- // {
- // if (retryCount == 5)
- // {
- // throw;
- // }
- //#if !NETSTANDARD
- // try
- // {
- //#endif
- // // Pause 5 msec
- // Thread.Sleep(5);
- //#if !NETSTANDARD
- // }
- // catch (ThreadInterruptedException ie)
- // {
- // var ex = new ThreadInterruptedException(ie.ToString(), ie);
- // //ex.AddSuppressed(exc);
- // throw ex;
- // }
- //#endif
- // }
- // finally
- // {
- // if (file != null)
- // {
- // file.Dispose();
- // }
- // }
-
- // if (success)
- // {
- // return;
- // }
-
- // retryCount++;
- // }
- // }
+ /// <summary>
+ /// Ensure that any writes to the given file is written to the storage device that contains it. </summary>
+ /// <param name="fileToSync"> The file to fsync </param>
+ /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s,
+ /// because not all file systems and operating systems allow to fsync on a directory) </param>
+ public static void Fsync(string fileToSync, bool isDir)
+ {
+ // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version
+ // they catch this in IOException branch and return if the call is for the directory.
+ // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by
+ // IOException block. No need to even attempt to fsync, just return if the call is for directory
+ if (isDir)
+ {
+ return;
+ }
+
+ var retryCount = 1;
+ while (true)
+ {
+ FileStream file = null;
+ bool success = false;
+ try
+ {
+ // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
+ // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
+ file = new FileStream(fileToSync,
+ FileMode.Open, // We shouldn't create a file when syncing.
+ // Java version uses FileChannel which doesn't create the file if it doesn't already exist,
+ // so there should be no reason for attempting to create it in Lucene.Net.
+ FileAccess.Write,
+ FileShare.ReadWrite);
+ //FileSupport.Sync(file);
+ file.Flush(true);
+ success = true;
+ }
+#pragma warning disable 168
+ catch (IOException e)
+#pragma warning restore 168
+ {
+ if (retryCount == 5)
+ {
+ throw;
+ }
+#if !NETSTANDARD
+ try
+ {
+#endif
+ // Pause 5 msec
+ Thread.Sleep(5);
+#if !NETSTANDARD
+ }
+ catch (ThreadInterruptedException ie)
+ {
+ var ex = new ThreadInterruptedException(ie.ToString(), ie);
+ //ex.AddSuppressed(exc);
+ throw ex;
+ }
+#endif
+ }
+ finally
+ {
+ if (file != null)
+ {
+ file.Dispose();
+ }
+ }
+
+ if (success)
+ {
+ return;
+ }
+
+ retryCount++;
+ }
+ }
}
}
\ No newline at end of file
[2/2] lucenenet git commit: Lucene.Net.Util.IOUtils: Added support
for .NET Standard 2.0 to process any ThreadInterruptExceptions the same way
as .NET Framework
Posted by ni...@apache.org.
Lucene.Net.Util.IOUtils: Added support for .NET Standard 2.0 to process any ThreadInterruptExceptions the same way as .NET Framework
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/8eada798
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/8eada798
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/8eada798
Branch: refs/heads/master
Commit: 8eada7984fbb9b47aceb11e18b4e188bf50c4f36
Parents: 681430e
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Thu Sep 14 04:29:17 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Thu Sep 14 04:29:17 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Util/IOUtils.cs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8eada798/src/Lucene.Net/Util/IOUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs
index 5907b60..9243104 100644
--- a/src/Lucene.Net/Util/IOUtils.cs
+++ b/src/Lucene.Net/Util/IOUtils.cs
@@ -565,18 +565,18 @@ namespace Lucene.Net.Util
{
throw;
}
-#if !NETSTANDARD
+#if !NETSTANDARD1_5
try
{
#endif
// Pause 5 msec
Thread.Sleep(5);
-#if !NETSTANDARD
+#if !NETSTANDARD1_5
}
catch (ThreadInterruptedException ie)
{
var ex = new ThreadInterruptedException(ie.ToString(), ie);
- //ex.AddSuppressed(exc);
+ ex.AddSuppressed(e);
throw ex;
}
#endif