You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/09/15 04:02:50 UTC
[1/5] lucenenet git commit: BUG: Lucene.Net.TestFramework: Having
sequential folder names creates situations where multiple threads are doing
operations on the same folder at the same time. Changed implementation to use
GetRandomFileName() to append a ra
Repository: lucenenet
Updated Branches:
refs/heads/master 6a95ad434 -> 34ef2ce56
BUG: Lucene.Net.TestFramework: Having sequential folder names creates situations where multiple threads are doing operations on the same folder at the same time. Changed implementation to use GetRandomFileName() to append a random string instead of an incremental number.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/4fdf6723
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/4fdf6723
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/4fdf6723
Branch: refs/heads/master
Commit: 4fdf672310ddbc7414956a3f156307184ef342a2
Parents: 6a95ad4
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Sep 15 07:24:01 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Sep 15 07:24:01 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4fdf6723/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
index 1c746ce..b23d0a1 100644
--- a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
+++ b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
@@ -2751,7 +2751,10 @@ namespace Lucene.Net.Util
{
throw new Exception("Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " + System.IO.Path.GetTempPath());
}
- f = new DirectoryInfo(Path.Combine(System.IO.Path.GetTempPath(), "LuceneTemp", prefix + "-" + attempt));
+ // LUCENENET specific - need to use a random file name instead of a sequential one or two threads may attempt to do
+ // two operations on a file at the same time.
+ //f = new DirectoryInfo(Path.Combine(System.IO.Path.GetTempPath(), "LuceneTemp", prefix + "-" + attempt));
+ f = new DirectoryInfo(Path.Combine(System.IO.Path.GetTempPath(), "LuceneTemp", prefix + "-" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName())));
try
{
[5/5] lucenenet git commit: API:
Lucene.Net.Replicator.Http.ReplicationService: Changed constructor parameter
to accept IReadOnlyDictionary rather than IDictionary
Posted by ni...@apache.org.
API: Lucene.Net.Replicator.Http.ReplicationService: Changed constructor parameter to accept IReadOnlyDictionary rather than IDictionary
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/34ef2ce5
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/34ef2ce5
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/34ef2ce5
Branch: refs/heads/master
Commit: 34ef2ce56c48beeb36e3a608c5339b84b36a5414
Parents: 1405926
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Sep 15 10:53:23 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Sep 15 10:53:23 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Replicator/Http/ReplicationService.cs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/34ef2ce5/src/Lucene.Net.Replicator/Http/ReplicationService.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Replicator/Http/ReplicationService.cs b/src/Lucene.Net.Replicator/Http/ReplicationService.cs
index b94241a..6ef6155 100644
--- a/src/Lucene.Net.Replicator/Http/ReplicationService.cs
+++ b/src/Lucene.Net.Replicator/Http/ReplicationService.cs
@@ -90,9 +90,9 @@ namespace Lucene.Net.Replicator.Http
private const int SHARD_IDX = 0, ACTION_IDX = 1;
private readonly string context;
- private readonly IDictionary<string, IReplicator> replicators;
+ private readonly IReadOnlyDictionary<string, IReplicator> replicators;
- public ReplicationService(IDictionary<string, IReplicator> replicators, string context = REPLICATION_CONTEXT)
+ public ReplicationService(IReadOnlyDictionary<string, IReplicator> replicators, string context = REPLICATION_CONTEXT)
{
this.context = context;
this.replicators = replicators;
[2/5] lucenenet git commit: BUG:
Lucene.Net.Store.NativeFSLockFactory: Missing lock on dictionary when
NativeFSLock.Dispose() is called. Also,
the lock needs to be removed from the cache whether it has opened a file or
not.
Posted by ni...@apache.org.
BUG: Lucene.Net.Store.NativeFSLockFactory: Missing lock on dictionary when NativeFSLock.Dispose() is called. Also, the lock needs to be removed from the cache whether it has opened a file or not.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/2bfa241d
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/2bfa241d
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/2bfa241d
Branch: refs/heads/master
Commit: 2bfa241d8913cf956d5fd28e0c374282f3363d07
Parents: 4fdf672
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Sep 15 09:01:13 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Sep 15 09:01:13 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Store/NativeFSLockFactory.cs | 43 ++++++++++++++----------
1 file changed, 26 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/2bfa241d/src/Lucene.Net/Store/NativeFSLockFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/NativeFSLockFactory.cs b/src/Lucene.Net/Store/NativeFSLockFactory.cs
index 8157e13..ca2a158 100644
--- a/src/Lucene.Net/Store/NativeFSLockFactory.cs
+++ b/src/Lucene.Net/Store/NativeFSLockFactory.cs
@@ -273,27 +273,36 @@ namespace Lucene.Net.Store
{
lock (this)
{
- if (channel != null)
+ // whether or not we have created a file, we need to remove
+ // the lock instance from the dictionary that tracks them.
+ try
{
- try
- {
+ lock (NativeFSLockFactory._locks)
NativeFSLockFactory._locks.Remove(path);
- }
- finally
+ }
+ finally
+ {
+ if (channel != null)
{
- IOUtils.DisposeWhileHandlingException(channel);
- channel = null;
- }
+ try
+ {
+ IOUtils.DisposeWhileHandlingException(channel);
+ }
+ finally
+ {
+ channel = null;
+ }
#if FEATURE_FILESTREAM_LOCK
- // try to delete the file if we created it, but it's not an error if we can't.
- try
- {
- File.Delete(path);
- }
- catch
- {
- }
+ // try to delete the file if we created it, but it's not an error if we can't.
+ try
+ {
+ File.Delete(path);
+ }
+ catch
+ {
+ }
#endif
+ }
}
}
}
@@ -339,4 +348,4 @@ namespace Lucene.Net.Store
return "NativeFSLock@" + path;
}
}
-}
+}
\ No newline at end of file
[3/5] lucenenet git commit: Revert Revert
"Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles
variable and all references to them"
Posted by ni...@apache.org.
Revert Revert "Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles variable and all references to them"
This reverts commit 681430e44ce316e18b788598ed002f6f14dd3822
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/b7c5f0b0
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/b7c5f0b0
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/b7c5f0b0
Branch: refs/heads/master
Commit: b7c5f0b081e31aa873bf375c1e75cbda434bef9f
Parents: 2bfa241
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Sep 15 09:35:27 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Sep 15 09:36:36 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Store/FSDirectory.cs | 55 ++++++-----
src/Lucene.Net/Util/IOUtils.cs | 153 ++++++++++++++++---------------
2 files changed, 112 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b7c5f0b0/src/Lucene.Net/Store/FSDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs
index 5487b96..8c65d9c 100644
--- a/src/Lucene.Net/Store/FSDirectory.cs
+++ b/src/Lucene.Net/Store/FSDirectory.cs
@@ -92,7 +92,10 @@ namespace Lucene.Net.Store
public const int DEFAULT_READ_CHUNK_SIZE = 8192;
protected readonly DirectoryInfo m_directory; // The underlying filesystem directory
- protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed
+
+ // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
+ // our FileStream is disposed.
+ //protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed
#pragma warning disable 612, 618
private int chunkSize = DEFAULT_READ_CHUNK_SIZE;
#pragma warning restore 612, 618
@@ -310,7 +313,9 @@ namespace Lucene.Net.Store
{
throw new IOException("Cannot delete " + file, e);
}
- m_staleFiles.Remove(name);
+ // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
+ // our FileStream is disposed.
+ //m_staleFiles.Remove(name);
}
/// <summary>
@@ -353,30 +358,35 @@ namespace Lucene.Net.Store
protected virtual void OnIndexOutputClosed(FSIndexOutput io)
{
- m_staleFiles.Add(io.name);
+ // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
+ // our FileStream is disposed.
+ //m_staleFiles.Add(io.name);
}
public override void Sync(ICollection<string> names)
{
EnsureOpen();
- ISet<string> toSync = new HashSet<string>(names);
- toSync.IntersectWith(m_staleFiles);
- // LUCENENET specific: Fsync breaks concurrency here.
- // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
- //foreach (var name in toSync)
+ // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before
+ // our FileStream is disposed. Therefore, there is nothing else to do in this method.
+ //ISet<string> toSync = new HashSet<string>(names);
+ //toSync.IntersectWith(m_staleFiles);
+
+ //// LUCENENET specific: Fsync breaks concurrency here.
+ //// Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+ ////foreach (var name in toSync)
+ ////{
+ //// Fsync(name);
+ ////}
+
+ //// fsync the directory itsself, but only if there was any file fsynced before
+ //// (otherwise it can happen that the directory does not yet exist)!
+ //if (toSync.Count > 0)
//{
- // Fsync(name);
+ // IOUtils.Fsync(m_directory.FullName, true);
//}
- // fsync the directory itsself, but only if there was any file fsynced before
- // (otherwise it can happen that the directory does not yet exist)!
- if (toSync.Count > 0)
- {
- IOUtils.Fsync(m_directory.FullName, true);
- }
-
- m_staleFiles.ExceptWith(toSync);
+ //m_staleFiles.ExceptWith(toSync);
}
public override string GetLockID()
@@ -535,9 +545,12 @@ namespace Lucene.Net.Store
// LUCENENET NOTE: FileStream doesn't have a way to set length
}
- protected virtual void Fsync(string name)
- {
- IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);
- }
+ // LUCENENET specific: Fsync is pointless in .NET, since we are
+ // calling FileStream.Flush(true) before the stream is disposed
+ // which means we never need it at the point in Java where it is called.
+ //protected virtual void Fsync(string name)
+ //{
+ // IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);
+ //}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b7c5f0b0/src/Lucene.Net/Util/IOUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs
index 9243104..125084f 100644
--- a/src/Lucene.Net/Util/IOUtils.cs
+++ b/src/Lucene.Net/Util/IOUtils.cs
@@ -522,80 +522,83 @@ namespace Lucene.Net.Util
}
}
- /// <summary>
- /// Ensure that any writes to the given file is written to the storage device that contains it. </summary>
- /// <param name="fileToSync"> The file to fsync </param>
- /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s,
- /// because not all file systems and operating systems allow to fsync on a directory) </param>
- public static void Fsync(string fileToSync, bool isDir)
- {
- // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version
- // they catch this in IOException branch and return if the call is for the directory.
- // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by
- // IOException block. No need to even attempt to fsync, just return if the call is for directory
- if (isDir)
- {
- return;
- }
-
- var retryCount = 1;
- while (true)
- {
- FileStream file = null;
- bool success = false;
- try
- {
- // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
- // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
- file = new FileStream(fileToSync,
- FileMode.Open, // We shouldn't create a file when syncing.
- // Java version uses FileChannel which doesn't create the file if it doesn't already exist,
- // so there should be no reason for attempting to create it in Lucene.Net.
- FileAccess.Write,
- FileShare.ReadWrite);
- //FileSupport.Sync(file);
- file.Flush(true);
- success = true;
- }
-#pragma warning disable 168
- catch (IOException e)
-#pragma warning restore 168
- {
- if (retryCount == 5)
- {
- throw;
- }
-#if !NETSTANDARD1_5
- try
- {
-#endif
- // Pause 5 msec
- Thread.Sleep(5);
-#if !NETSTANDARD1_5
- }
- catch (ThreadInterruptedException ie)
- {
- var ex = new ThreadInterruptedException(ie.ToString(), ie);
- ex.AddSuppressed(e);
- throw ex;
- }
-#endif
- }
- finally
- {
- if (file != null)
- {
- file.Dispose();
- }
- }
-
- if (success)
- {
- return;
- }
-
- retryCount++;
- }
- }
+ // LUCENENET specific: Fsync is pointless in .NET, since we are
+ // calling FileStream.Flush(true) before the stream is disposed
+ // which means we never need it at the point in Java where it is called.
+ // /// <summary>
+ // /// Ensure that any writes to the given file is written to the storage device that contains it. </summary>
+ // /// <param name="fileToSync"> The file to fsync </param>
+ // /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s,
+ // /// because not all file systems and operating systems allow to fsync on a directory) </param>
+ // public static void Fsync(string fileToSync, bool isDir)
+ // {
+ // // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version
+ // // they catch this in IOException branch and return if the call is for the directory.
+ // // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by
+ // // IOException block. No need to even attempt to fsync, just return if the call is for directory
+ // if (isDir)
+ // {
+ // return;
+ // }
+
+ // var retryCount = 1;
+ // while (true)
+ // {
+ // FileStream file = null;
+ // bool success = false;
+ // try
+ // {
+ // // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
+ // // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
+ // file = new FileStream(fileToSync,
+ // FileMode.Open, // We shouldn't create a file when syncing.
+ // // Java version uses FileChannel which doesn't create the file if it doesn't already exist,
+ // // so there should be no reason for attempting to create it in Lucene.Net.
+ // FileAccess.Write,
+ // FileShare.ReadWrite);
+ // //FileSupport.Sync(file);
+ // file.Flush(true);
+ // success = true;
+ // }
+ //#pragma warning disable 168
+ // catch (IOException e)
+ //#pragma warning restore 168
+ // {
+ // if (retryCount == 5)
+ // {
+ // throw;
+ // }
+ //#if !NETSTANDARD1_5
+ // try
+ // {
+ //#endif
+ // // Pause 5 msec
+ // Thread.Sleep(5);
+ //#if !NETSTANDARD1_5
+ // }
+ // catch (ThreadInterruptedException ie)
+ // {
+ // var ex = new ThreadInterruptedException(ie.ToString(), ie);
+ // ex.AddSuppressed(e);
+ // throw ex;
+ // }
+ //#endif
+ // }
+ // finally
+ // {
+ // if (file != null)
+ // {
+ // file.Dispose();
+ // }
+ // }
+
+ // if (success)
+ // {
+ // return;
+ // }
+
+ // retryCount++;
+ // }
+ // }
}
}
\ No newline at end of file
[4/5] lucenenet git commit: BUG:
Lucene.Net.Replicator.LocalReplicator.ObtainFile(): Fixed
KeyNotFoundException issue with sessions dictionary.
Posted by ni...@apache.org.
BUG: Lucene.Net.Replicator.LocalReplicator.ObtainFile(): Fixed KeyNotFoundException issue with sessions dictionary.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/14059266
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/14059266
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/14059266
Branch: refs/heads/master
Commit: 140592662d48825aa48775b8b1ae70c130203eb2
Parents: b7c5f0b
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Sep 15 10:41:41 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Sep 15 10:41:41 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Replicator/LocalReplicator.cs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/14059266/src/Lucene.Net.Replicator/LocalReplicator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Replicator/LocalReplicator.cs b/src/Lucene.Net.Replicator/LocalReplicator.cs
index 349a088..5e6983a 100644
--- a/src/Lucene.Net.Replicator/LocalReplicator.cs
+++ b/src/Lucene.Net.Replicator/LocalReplicator.cs
@@ -237,8 +237,8 @@ namespace Lucene.Net.Replicator
{
EnsureOpen();
- ReplicationSession session = sessions[sessionId];
- if (session != null && session.IsExpired(ExpirationThreshold))
+ ReplicationSession session;
+ if (sessions.TryGetValue(sessionId, out session) && session != null && session.IsExpired(ExpirationThreshold))
{
ReleaseSession(sessionId);
session = null;