You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by do...@apache.org on 2009/07/29 20:04:24 UTC
svn commit: r798995 [5/35] - in /incubator/lucene.net/trunk/C#/src:
Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/
Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/
Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net/...
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs Wed Jul 29 18:04:12 2009
@@ -15,432 +15,458 @@
* limitations under the License.
*/
-using System;
-
using Directory = Lucene.Net.Store.Directory;
namespace Lucene.Net.Index
{
-
- /// <summary>A {@link MergeScheduler} that runs each merge using a
- /// separate thread, up until a maximum number of threads
- /// ({@link #setMaxThreadCount}) at which points merges are
- /// run in the foreground, serially. This is a simple way
- /// to use concurrency in the indexing process without
- /// having to create and manage application level
- /// threads.
- /// </summary>
-
- public class ConcurrentMergeScheduler : MergeScheduler
- {
-
- private int mergeThreadPriority = - 1;
-
- private System.Collections.IList mergeThreads = new System.Collections.ArrayList();
- private int maxThreadCount = 3;
-
- private System.Collections.IList exceptions = new System.Collections.ArrayList();
- private Directory dir;
-
- private bool closed;
- private IndexWriter writer;
-
- public ConcurrentMergeScheduler()
- {
- if (allInstances != null)
- {
- // Only for testing
- AddMyself();
- }
- }
-
- /// <summary>Sets the max # simultaneous threads that may be
- /// running. If a merge is necessary yet we already have
- /// this many threads running, the merge is returned back
- /// to IndexWriter so that it runs in the "foreground".
- /// </summary>
- public virtual void SetMaxThreadCount(int count)
- {
- if (count < 1)
- throw new System.ArgumentException("count should be at least 1");
- maxThreadCount = count;
- }
-
- /// <summary>Get the max # simultaneous threads that may be</summary>
- /// <seealso cref="setMaxThreadCount.">
- /// </seealso>
- public virtual int GetMaxThreadCount()
- {
- return maxThreadCount;
- }
-
- /// <summary>Return the priority that merge threads run at. By
- /// default the priority is 1 plus the priority of (ie,
- /// slightly higher priority than) the first thread that
- /// calls merge.
- /// </summary>
- public virtual int GetMergeThreadPriority()
- {
- lock (this)
- {
- InitMergeThreadPriority();
- return mergeThreadPriority;
- }
- }
-
- /// <summary>Return the priority that merge threads run at. </summary>
- public virtual void SetMergeThreadPriority(int pri)
- {
- lock (this)
- {
- if (pri > (int) System.Threading.ThreadPriority.Highest || pri < (int) System.Threading.ThreadPriority.Lowest)
- throw new System.ArgumentException("priority must be in range " + (int) System.Threading.ThreadPriority.Lowest + " .. " + (int) System.Threading.ThreadPriority.Highest + " inclusive");
- mergeThreadPriority = pri;
-
- int numThreads = MergeThreadCount();
- for (int i = 0; i < numThreads; i++)
- {
- MergeThread merge = (MergeThread) mergeThreads[i];
- merge.SetThreadPriority(pri);
- }
- }
- }
-
- private void Message(System.String message)
- {
- if (writer != null)
- writer.Message("CMS: " + message);
- }
-
- private void InitMergeThreadPriority()
- {
- lock (this)
- {
- if (mergeThreadPriority == - 1)
- {
- // Default to slightly higher priority than our
- // calling thread
- mergeThreadPriority = 1 + (System.Int32) SupportClass.ThreadClass.Current().Priority;
- if (mergeThreadPriority > (int) System.Threading.ThreadPriority.Highest)
- mergeThreadPriority = (int) System.Threading.ThreadPriority.Highest;
- }
- }
- }
-
- public override void Close()
- {
- closed = true;
- }
-
- public virtual void Sync()
- {
- lock (this)
- {
- while (MergeThreadCount() > 0)
- {
- Message("now wait for threads; currently " + mergeThreads.Count + " still running");
- int count = mergeThreads.Count;
- for (int i = 0; i < count; i++)
- Message(" " + i + ": " + ((MergeThread) mergeThreads[i]));
-
- try
- {
- System.Threading.Monitor.Wait(this);
- }
- catch (System.Threading.ThreadInterruptedException)
- {
- }
- }
- }
- }
-
- private int MergeThreadCount()
- {
- lock (this)
- {
- int count = 0;
- int numThreads = mergeThreads.Count;
- for (int i = 0; i < numThreads; i++)
- if (((MergeThread) mergeThreads[i]).IsAlive)
- count++;
- return count;
- }
- }
-
- public override void Merge(IndexWriter writer)
- {
-
- this.writer = writer;
-
- InitMergeThreadPriority();
-
- dir = writer.GetDirectory();
-
- // First, quickly run through the newly proposed merges
- // and add any orthogonal merges (ie a merge not
- // involving segments already pending to be merged) to
- // the queue. If we are way behind on merging, many of
- // these newly proposed merges will likely already be
- // registered.
-
- Message("now merge");
- Message(" index: " + writer.SegString());
-
- // Iterate, pulling from the IndexWriter's queue of
- // pending merges, until its empty:
- while (true)
- {
-
- // TODO: we could be careful about which merges to do in
- // the BG (eg maybe the "biggest" ones) vs FG, which
- // merges to do first (the easiest ones?), etc.
-
- MergePolicy.OneMerge merge = writer.GetNextMerge();
- if (merge == null)
- {
- Message(" no more merges pending; now return");
- return ;
- }
-
- // We do this w/ the primary thread to keep
- // deterministic assignment of segment names
- writer.MergeInit(merge);
-
- Message(" consider merge " + merge.SegString(dir));
-
- if (merge.isExternal)
- {
- Message(" merge involves segments from an external directory; now run in foreground");
- }
- else
- {
- lock (this)
- {
- if (MergeThreadCount() < maxThreadCount)
- {
- // OK to spawn a new merge thread to handle this
- // merge:
- MergeThread merger = new MergeThread(this, writer, merge);
- mergeThreads.Add(merger);
- Message(" launch new thread [" + merger.Name + "]");
- merger.SetThreadPriority(mergeThreadPriority);
- merger.IsBackground = true;
- merger.Start();
- continue;
- }
- else
- Message(" too many merge threads running; run merge in foreground");
- }
- }
-
- // Too many merge threads already running, so we do
- // this in the foreground of the calling thread
- writer.Merge(merge);
- }
- }
-
- private class MergeThread:SupportClass.ThreadClass
- {
- private void InitBlock(ConcurrentMergeScheduler enclosingInstance)
- {
- this.enclosingInstance = enclosingInstance;
- }
- private ConcurrentMergeScheduler enclosingInstance;
- public ConcurrentMergeScheduler Enclosing_Instance
- {
- get
- {
- return enclosingInstance;
- }
-
- }
-
- internal IndexWriter writer;
- internal MergePolicy.OneMerge startMerge;
- internal MergePolicy.OneMerge runningMerge;
-
- public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
- {
- InitBlock(enclosingInstance);
- this.writer = writer;
- this.startMerge = startMerge;
- }
-
- public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
- {
- lock (this)
- {
- runningMerge = merge;
- }
- }
-
- public virtual MergePolicy.OneMerge GetRunningMerge()
- {
- lock (this)
- {
- return runningMerge;
- }
- }
-
- public virtual void SetThreadPriority(int pri)
- {
- try
- {
- Priority = (System.Threading.ThreadPriority) pri;
- }
- catch (System.NullReferenceException)
- {
- // Strangely, Sun's JDK 1.5 on Linux sometimes
- // throws NPE out of here...
- }
- catch (System.Security.SecurityException)
- {
- // Ignore this because we will still run fine with
- // normal thread priority
- }
- }
-
- override public void Run()
- {
-
- // First time through the while loop we do the merge
- // that we were started with:
- MergePolicy.OneMerge merge = this.startMerge;
-
- try
- {
-
- Enclosing_Instance.Message(" merge thread: start");
-
- while (true)
- {
- SetRunningMerge(merge);
- writer.Merge(merge);
-
- // Subsequent times through the loop we do any new
- // merge that writer says is necessary:
- merge = writer.GetNextMerge();
- if (merge != null)
- {
- writer.MergeInit(merge);
- Enclosing_Instance.Message(" merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
- }
- else
- break;
- }
-
- Enclosing_Instance.Message(" merge thread: done");
- }
- catch (System.Exception exc)
- {
-
- if (merge != null)
- {
- merge.SetException(exc);
- writer.AddMergeException(merge);
- }
-
- // Ignore the exception if it was due to abort:
- if (!(exc is MergePolicy.MergeAbortedException))
- {
- lock (Enclosing_Instance)
- {
- Enclosing_Instance.exceptions.Add(exc);
- }
-
- if (!Enclosing_Instance.suppressExceptions)
- {
- // suppressExceptions is normally only set during
- // testing.
- Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
- throw new MergePolicy.MergeException(exc);
- }
- }
- }
- finally
- {
- lock (Enclosing_Instance)
- {
- Enclosing_Instance.mergeThreads.Remove(this);
- System.Threading.Monitor.PulseAll(Enclosing_Instance);
- }
- }
- }
-
- public override System.String ToString()
- {
- MergePolicy.OneMerge merge = GetRunningMerge();
- if (merge == null)
- merge = startMerge;
- return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
- }
- }
-
- internal static bool anyExceptions = false;
-
- /// <summary>Used for testing </summary>
- public static bool AnyUnhandledExceptions()
- {
- lock (allInstances.SyncRoot)
- {
- int count = allInstances.Count;
- // Make sure all outstanding threads are done so we see
- // any exceptions they may produce:
- for (int i = 0; i < count; i++)
- ((ConcurrentMergeScheduler) allInstances[i]).Sync();
- return anyExceptions;
- }
- }
-
- /// <summary>Used for testing </summary>
- private void AddMyself()
- {
- lock (allInstances.SyncRoot)
- {
- int size = 0;
- int upto = 0;
- for (int i = 0; i < size; i++)
- {
- ConcurrentMergeScheduler other = (ConcurrentMergeScheduler) allInstances[i];
- if (!(other.closed && 0 == other.MergeThreadCount()))
- // Keep this one for now: it still has threads or
- // may spawn new threads
- allInstances[upto++] = other;
- }
- ((System.Collections.IList) ((System.Collections.ArrayList) allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
- allInstances.Add(this);
- }
- }
-
- private bool suppressExceptions;
-
- /// <summary>Used for testing </summary>
- internal virtual void SetSuppressExceptions()
- {
- suppressExceptions = true;
- }
-
- /// <summary>Used for testing </summary>
- internal virtual void ClearSuppressExceptions()
- {
- suppressExceptions = false;
- }
-
- /// <summary>Used for testing </summary>
- private static System.Collections.IList allInstances;
- public static void SetTestMode()
- {
- allInstances = new System.Collections.ArrayList();
- }
+ /// <summary>
+ /// A {@link MergeScheduler} that runs each merge using a
+ /// separate thread, up until a maximum number of threads
+ /// ({@link #setMaxThreadCount}) at which when a merge is
+ /// needed, the thread(s) that are updating the index will
+ /// pause until one or more merges completes. This is a
+ /// simple way to use concurrency in the indexing process
+ /// without having to create and manage application level
+ /// threads.
+ /// </summary>
+ public class ConcurrentMergeScheduler : MergeScheduler
+ {
+
+ private int mergeThreadPriority = -1;
+
+ protected System.Collections.Generic.List<MergeThread> mergeThreads = new System.Collections.Generic.List<MergeThread>();
+
+ // max number of threads allowed to be merging at once
+ private int maxThreadCount = 3;
+
+ private System.Collections.Generic.List<System.Exception> exceptions = new System.Collections.Generic.List<System.Exception>();
+ protected Directory dir;
+
+ private bool closed;
+ protected IndexWriter writer;
+ protected int mergeThreadCount;
+
+ public ConcurrentMergeScheduler()
+ {
+ if (allInstances != null)
+ {
+ // Only for testing
+ AddMyself();
+ }
+ }
+
+ /// <summary>Sets the max # simultaneous threads that may be
+ /// running. If a merge is necessary yet we already have
+ /// this many threads running, the incoming thread (that
+ /// is calling add/updateDocument) will block until
+ /// a merge thread has completed.
+ /// </summary>
+ public virtual void SetMaxThreadCount(int count)
+ {
+ if (count < 1)
+ throw new System.ArgumentException("count should be at least 1");
+ maxThreadCount = count;
+ }
+
+ /// <summary>Get the max # simultaneous threads that may be</summary>
+ /// <seealso cref="setMaxThreadCount.">
+ /// </seealso>
+ public virtual int GetMaxThreadCount()
+ {
+ return maxThreadCount;
+ }
+
+ /// <summary>Return the priority that merge threads run at. By
+ /// default the priority is 1 plus the priority of (ie,
+ /// slightly higher priority than) the first thread that
+ /// calls merge.
+ /// </summary>
+ public virtual int GetMergeThreadPriority()
+ {
+ lock (this)
+ {
+ InitMergeThreadPriority();
+ return mergeThreadPriority;
+ }
+ }
+
+ /// <summary>Return the priority that merge threads run at. </summary>
+ public virtual void SetMergeThreadPriority(int pri)
+ {
+ lock (this)
+ {
+ if (pri > (int)System.Threading.ThreadPriority.Highest || pri < (int)System.Threading.ThreadPriority.Lowest)
+ throw new System.ArgumentException("priority must be in range " + (int)System.Threading.ThreadPriority.Lowest + " .. " + (int)System.Threading.ThreadPriority.Highest + " inclusive");
+ mergeThreadPriority = pri;
+
+ int numThreads = MergeThreadCount();
+ for (int i = 0; i < numThreads; i++)
+ {
+ MergeThread merge = mergeThreads[i];
+ merge.SetThreadPriority(pri);
+ }
+ }
+ }
+
+ private void Message(System.String message)
+ {
+ if (writer != null)
+ writer.Message("CMS: " + message);
+ }
+
+ private void InitMergeThreadPriority()
+ {
+ lock (this)
+ {
+ if (mergeThreadPriority == -1)
+ {
+ // Default to slightly higher priority than our calling thread
+ mergeThreadPriority = 1 + (int)System.Threading.Thread.CurrentThread.Priority;
+ if (mergeThreadPriority > (int)System.Threading.ThreadPriority.Highest)
+ mergeThreadPriority = (int)System.Threading.ThreadPriority.Highest;
+ }
+ }
+ }
+
+ public override void Close()
+ {
+ closed = true;
+ }
+
+ public virtual void Sync()
+ {
+ lock (this)
+ {
+ while (MergeThreadCount() > 0)
+ {
+ Message("now wait for threads; currently " + mergeThreads.Count + " still running");
+ int count = mergeThreads.Count;
+ for (int i = 0; i < count; i++)
+ Message(" " + i + ": " + mergeThreads[i]);
+
+ try
+ {
+ System.Threading.Monitor.Wait(this);
+ }
+ catch (System.Threading.ThreadInterruptedException)
+ {
+ }
+ }
+ }
+ }
+
+ private int MergeThreadCount()
+ {
+ lock (this)
+ {
+ int count = 0;
+ int numThreads = mergeThreads.Count;
+ for (int i = 0; i < numThreads; i++)
+ if (mergeThreads[i].IsAlive)
+ count++;
+ return count;
+ }
+ }
+
+ public override void Merge(IndexWriter writer)
+ {
+ this.writer = writer;
+ InitMergeThreadPriority();
+ dir = writer.GetDirectory();
+
+ // First, quickly run through the newly proposed merges
+ // and add any orthogonal merges (ie a merge not
+ // involving segments already pending to be merged) to
+ // the queue. If we are way behind on merging, many of
+ // these newly proposed merges will likely already be
+ // registered.
+
+ Message("now merge");
+ Message(" index: " + writer.SegString());
+
+ // Iterate, pulling from the IndexWriter's queue of
+ // pending merges, until its empty:
+ while (true)
+ {
+ // TODO: we could be careful about which merges to do in
+ // the BG (eg maybe the "biggest" ones) vs FG, which
+ // merges to do first (the easiest ones?), etc.
+
+ MergePolicy.OneMerge merge = writer.GetNextMerge();
+ if (merge == null)
+ {
+ Message(" no more merges pending; now return");
+ return;
+ }
+
+ // We do this w/ the primary thread to keep
+ // deterministic assignment of segment names
+ writer.MergeInit(merge);
+
+ lock (this)
+ {
+ while (MergeThreadCount() >= maxThreadCount)
+ {
+ Message(" too may merge threads running; stalling...");
+ try
+ {
+ System.Threading.Monitor.Wait(this);
+ }
+ catch (System.Threading.ThreadInterruptedException)
+ {
+ SupportClass.ThreadClass.Current().Interrupt();
+ }
+ }
+
+ Message(" consider merge " + merge.SegString(dir));
+
+ System.Diagnostics.Debug.Assert(MergeThreadCount() < maxThreadCount);
+
+ // OK to spawn a new merge thread to handle this
+ // merge:
+ MergeThread merger = GetMergeThread(writer, merge);
+ mergeThreads.Add(merger);
+ Message(" launch new thread [" + merger.Name + "]");
+ merger.Start();
+ }
+ }
+ }
+
+ /// <summary>
+ /// Does the acural merge, by calling IndexWriter.Merge().
+ /// </summary>
+ /// <param name="merge"></param>
+ virtual protected void DoMerge(MergePolicy.OneMerge merge)
+ {
+ writer.Merge(merge);
+ }
+
+ /// <summary>
+ /// Create and return a new MergeThread.
+ /// </summary>
+ /// <param name="writer"></param>
+ /// <param name="merge"></param>
+ /// <returns></returns>
+ virtual protected MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
+ {
+ MergeThread thread = new MergeThread(this, writer, merge);
+ thread.SetThreadPriority(mergeThreadPriority);
+ thread.IsBackground = true;
+ thread.Name = "Lucene Merge Thread #" + mergeThreadCount++;
+ return thread;
+ }
+
+ protected class MergeThread : SupportClass.ThreadClass
+ {
+ private void InitBlock(ConcurrentMergeScheduler enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private ConcurrentMergeScheduler enclosingInstance;
+ public ConcurrentMergeScheduler Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+
+ internal IndexWriter writer;
+ internal MergePolicy.OneMerge startMerge;
+ internal MergePolicy.OneMerge runningMerge;
+
+ public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
+ {
+ InitBlock(enclosingInstance);
+ this.writer = writer;
+ this.startMerge = startMerge;
+ }
+
+ public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
+ {
+ lock (this)
+ {
+ runningMerge = merge;
+ }
+ }
+
+ public virtual MergePolicy.OneMerge GetRunningMerge()
+ {
+ lock (this)
+ {
+ return runningMerge;
+ }
+ }
+
+ public virtual void SetThreadPriority(int pri)
+ {
+ try
+ {
+ Priority = (System.Threading.ThreadPriority)pri;
+ }
+ catch (System.NullReferenceException)
+ {
+ // Strangely, Sun's JDK 1.5 on Linux sometimes
+ // throws NPE out of here...
+ }
+ catch (System.Security.SecurityException)
+ {
+ // Ignore this because we will still run fine with
+ // normal thread priority
+ }
+ }
+
+ override public void Run()
+ {
+
+ // First time through the while loop we do the merge
+ // that we were started with:
+ MergePolicy.OneMerge merge = this.startMerge;
+
+ try
+ {
+
+ Enclosing_Instance.Message(" merge thread: start");
+
+ while (true)
+ {
+ SetRunningMerge(merge);
+ Enclosing_Instance.DoMerge(merge);
+
+ // Subsequent times through the loop we do any new
+ // merge that writer says is necessary:
+ merge = writer.GetNextMerge();
+ if (merge != null)
+ {
+ writer.MergeInit(merge);
+ Enclosing_Instance.Message(" merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
+ }
+ else
+ break;
+ }
+
+ Enclosing_Instance.Message(" merge thread: done");
+ }
+ catch (System.Exception exc)
+ {
+ // Ignore the exception if it was due to abort:
+ if (!(exc is MergePolicy.MergeAbortedException))
+ {
+ lock (Enclosing_Instance)
+ {
+ Enclosing_Instance.exceptions.Add(exc);
+ }
+
+ if (!Enclosing_Instance.suppressExceptions)
+ {
+ // suppressExceptions is normally only set during
+ // testing.
+ Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
+ Enclosing_Instance.HandleMergeException(exc);
+ }
+ }
+ }
+ finally
+ {
+ lock (Enclosing_Instance)
+ {
+ System.Threading.Monitor.PulseAll(Enclosing_Instance);
+ bool removed = Enclosing_Instance.mergeThreads.Remove(this);
+ System.Diagnostics.Debug.Assert(removed);
+ }
+ }
+ }
+
+ public override System.String ToString()
+ {
+ MergePolicy.OneMerge merge = GetRunningMerge();
+ if (merge == null)
+ merge = startMerge;
+ return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
+ }
+ }
+
+ virtual protected void HandleMergeException(System.Exception exc)
+ {
+ throw new MergePolicy.MergeException(exc, dir);
+ }
+
+ internal static bool anyExceptions = false;
+
+ /// <summary>Used for testing </summary>
+ public static bool AnyUnhandledExceptions()
+ {
+ lock (allInstances.SyncRoot)
+ {
+ int count = allInstances.Count;
+ // Make sure all outstanding threads are done so we see
+ // any exceptions they may produce:
+ for (int i = 0; i < count; i++)
+ ((ConcurrentMergeScheduler)allInstances[i]).Sync();
+ bool v = anyExceptions;
+ anyExceptions = false;
+ return v;
+ }
+ }
+
+ public static void ClearUnhandledExceptions()
+ {
+ lock (allInstances)
+ {
+ anyExceptions = false;
+ }
+ }
+
+ /// <summary>Used for testing </summary>
+ private void AddMyself()
+ {
+ lock (allInstances.SyncRoot)
+ {
+ int size = 0;
+ int upto = 0;
+ for (int i = 0; i < size; i++)
+ {
+ ConcurrentMergeScheduler other = (ConcurrentMergeScheduler)allInstances[i];
+ if (!(other.closed && 0 == other.MergeThreadCount()))
+ // Keep this one for now: it still has threads or
+ // may spawn new threads
+ allInstances[upto++] = other;
+ }
+ ((System.Collections.IList)((System.Collections.ArrayList)allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
+ allInstances.Add(this);
+ }
+ }
+
+ private bool suppressExceptions;
+
+ /// <summary>Used for testing </summary>
+ internal virtual void SetSuppressExceptions()
+ {
+ suppressExceptions = true;
+ }
+
+ /// <summary>Used for testing </summary>
+ internal virtual void ClearSuppressExceptions()
+ {
+ suppressExceptions = false;
+ }
+
+ /// <summary>Used for testing </summary>
+ private static System.Collections.IList allInstances;
+ public static void SetTestMode()
+ {
+ allInstances = new System.Collections.ArrayList();
+ }
public void SetSuppressExceptions_ForNUnitTest()
{
SetSuppressExceptions();
}
-
+
public void ClearSuppressExceptions_ForNUnitTest()
{
ClearSuppressExceptions();
}
- }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListWriter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs Wed Jul 29 18:04:12 2009
@@ -61,16 +61,18 @@
this.curStorePayloads = storePayloads;
this.curPayloadLength = payloadLength;
this.curFreqPointer = freqOutput.GetFilePointer();
- this.curProxPointer = proxOutput.GetFilePointer();
+ if (proxOutput != null)
+ this.curProxPointer = proxOutput.GetFilePointer();
}
protected internal override void ResetSkip()
{
base.ResetSkip();
- for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0;
- for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list
- for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.GetFilePointer();
- for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.GetFilePointer();
+ SupportClass.CollectionsSupport.ArrayFill(lastSkipDoc, 0);
+ SupportClass.CollectionsSupport.ArrayFill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list
+ SupportClass.CollectionsSupport.ArrayFill(lastSkipFreqPointer, freqOutput.GetFilePointer());
+ if (proxOutput != null)
+ SupportClass.CollectionsSupport.ArrayFill(lastSkipProxPointer, proxOutput.GetFilePointer());
}
protected internal override void WriteSkipData(int level, IndexOutput skipBuffer)
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DirectoryIndexReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs Wed Jul 29 18:04:12 2009
@@ -16,6 +16,7 @@
*/
using System;
+using System.Collections.Generic;
using Directory = Lucene.Net.Store.Directory;
using Lock = Lucene.Net.Store.Lock;
@@ -23,357 +24,529 @@
namespace Lucene.Net.Index
{
-
- /// <summary> IndexReader implementation that has access to a Directory.
- /// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
- /// "own" the directory, which means that they try to acquire a write lock
- /// whenever index modifications are performed.
- /// </summary>
- abstract public class DirectoryIndexReader : IndexReader
- {
- private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
- {
- private void InitBlock(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy)
- {
- this.closeDirectory = closeDirectory;
- this.deletionPolicy = deletionPolicy;
- }
- private bool closeDirectory;
- private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
- internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1) : base(Param1)
- {
- InitBlock(closeDirectory, deletionPolicy);
- }
-
- protected internal override System.Object DoBody(System.String segmentFileName)
- {
-
- SegmentInfos infos = new SegmentInfos();
- infos.Read(directory, segmentFileName);
-
- DirectoryIndexReader reader;
-
- if (infos.Count == 1)
- {
- // index is optimized
- reader = SegmentReader.Get(infos, infos.Info(0), closeDirectory);
- }
- else
- {
- reader = new MultiSegmentReader(directory, infos, closeDirectory);
- }
- reader.SetDeletionPolicy(deletionPolicy);
- return reader;
- }
- }
-
- private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
- {
- private void InitBlock(DirectoryIndexReader enclosingInstance)
- {
- this.enclosingInstance = enclosingInstance;
- }
- private DirectoryIndexReader enclosingInstance;
- public DirectoryIndexReader Enclosing_Instance
- {
- get
- {
- return enclosingInstance;
- }
-
- }
- internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1) : base(Param1)
- {
- InitBlock(enclosingInstance);
- }
-
- protected internal override System.Object DoBody(System.String segmentFileName)
- {
- SegmentInfos infos = new SegmentInfos();
- infos.Read(directory, segmentFileName);
-
- DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
-
- if (Enclosing_Instance != newReader)
- {
- newReader.Init(directory, infos, Enclosing_Instance.closeDirectory);
- newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
- }
-
- return newReader;
- }
- }
- protected internal Directory directory;
- protected internal bool closeDirectory;
- private IndexDeletionPolicy deletionPolicy;
-
- private SegmentInfos segmentInfos;
- private Lock writeLock;
- private bool stale;
-
- /// <summary>Used by commit() to record pre-commit state in case
- /// rollback is necessary
- /// </summary>
- private bool rollbackHasChanges;
- private SegmentInfos rollbackSegmentInfos;
-
-
- internal virtual void Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory)
- {
- this.directory = directory;
- this.segmentInfos = segmentInfos;
- this.closeDirectory = closeDirectory;
- }
-
- protected internal DirectoryIndexReader()
- {
- }
-
- internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory) : base()
- {
- Init(directory, segmentInfos, closeDirectory);
- }
-
- internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
- {
-
- return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory).Run();
- }
-
-
- public override IndexReader Reopen()
- {
- lock (this)
- {
- EnsureOpen();
-
- if (this.hasChanges || this.IsCurrent())
- {
- // the index hasn't changed - nothing to do here
- return this;
- }
-
- return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile1(this, directory).Run();
- }
- }
-
- /// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
- protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
-
- public virtual void SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
- {
- this.deletionPolicy = deletionPolicy;
- }
-
- /// <summary>Returns the directory this index resides in.</summary>
- public override Directory Directory()
- {
- EnsureOpen();
- return directory;
- }
-
- /// <summary> Version number when this IndexReader was opened.</summary>
- public override long GetVersion()
- {
- EnsureOpen();
- return segmentInfos.GetVersion();
- }
-
- /// <summary> Check whether this IndexReader is still using the
- /// current (i.e., most recently committed) version of the
- /// index. If a writer has committed any changes to the
- /// index since this reader was opened, this will return
- /// <code>false</code>, in which case you must open a new
- /// IndexReader in order to see the changes. See the
- /// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
- /// flag which controls when the {@link IndexWriter}
- /// actually commits changes to the index.
- ///
- /// </summary>
- /// <throws> CorruptIndexException if the index is corrupt </throws>
- /// <throws> IOException if there is a low-level IO error </throws>
- public override bool IsCurrent()
- {
- EnsureOpen();
- return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
- }
-
- /// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
- /// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
- /// </returns>
- public override bool IsOptimized()
- {
- EnsureOpen();
- return segmentInfos.Count == 1 && HasDeletions() == false;
- }
-
- protected internal override void DoClose()
- {
- if (closeDirectory)
- directory.Close();
- }
-
- /// <summary> Commit changes resulting from delete, undeleteAll, or
- /// setNorm operations
- ///
- /// If an exception is hit, then either no changes or all
- /// changes will have been committed to the index
- /// (transactional semantics).
- /// </summary>
- /// <throws> IOException if there is a low-level IO error </throws>
- protected internal override void DoCommit()
- {
- if (hasChanges)
- {
- if (segmentInfos != null)
- {
-
- // Default deleter (for backwards compatibility) is
- // KeepOnlyLastCommitDeleter:
- IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
-
- // Checkpoint the state we are about to change, in
- // case we have to roll back:
- StartCommit();
-
- bool success = false;
- try
- {
- CommitChanges();
- segmentInfos.Write(directory);
- success = true;
- }
- finally
- {
-
- if (!success)
- {
-
- // Rollback changes that were made to
- // SegmentInfos but failed to get [fully]
- // committed. This way this reader instance
- // remains consistent (matched to what's
- // actually in the index):
- RollbackCommit();
-
- // Recompute deletable files & remove them (so
- // partially written .del files, etc, are
- // removed):
- deleter.Refresh();
- }
- }
-
- // Have the deleter remove any now unreferenced
- // files due to this commit:
- deleter.Checkpoint(segmentInfos, true);
-
- if (writeLock != null)
- {
- writeLock.Release(); // release write lock
- writeLock = null;
- }
- }
- else
- CommitChanges();
- }
- hasChanges = false;
- }
-
- protected internal abstract void CommitChanges();
-
- /// <summary> Tries to acquire the WriteLock on this directory.
- /// this method is only valid if this IndexReader is directory owner.
- ///
- /// </summary>
- /// <throws> StaleReaderException if the index has changed </throws>
- /// <summary> since this reader was opened
- /// </summary>
- /// <throws> CorruptIndexException if the index is corrupt </throws>
- /// <throws> LockObtainFailedException if another writer </throws>
- /// <summary> has this index open (<code>write.lock</code> could not
- /// be obtained)
- /// </summary>
- /// <throws> IOException if there is a low-level IO error </throws>
- protected internal override void AcquireWriteLock()
- {
- if (segmentInfos != null)
- {
- EnsureOpen();
- if (stale)
- throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
-
- if (this.writeLock == null)
- {
- Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
- if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
- // obtain write lock
- {
- throw new LockObtainFailedException("Index locked for write: " + writeLock);
- }
- this.writeLock = writeLock;
-
- // we have to check whether index has changed since this reader was opened.
- // if so, this reader is no longer valid for deletion
- if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
- {
- stale = true;
- this.writeLock.Release();
- this.writeLock = null;
- throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
- }
- }
- }
- }
-
- /// <summary> Should internally checkpoint state that will change
- /// during commit so that we can rollback if necessary.
- /// </summary>
- internal virtual void StartCommit()
- {
- if (segmentInfos != null)
- {
- rollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
- }
- rollbackHasChanges = hasChanges;
- }
-
- /// <summary> Rolls back state to just before the commit (this is
- /// called by commit() if there is some exception while
- /// committing).
- /// </summary>
- internal virtual void RollbackCommit()
- {
- if (segmentInfos != null)
- {
- for (int i = 0; i < segmentInfos.Count; i++)
- {
- // Rollback each segmentInfo. Because the
- // SegmentReader holds a reference to the
- // SegmentInfo we can't [easily] just replace
- // segmentInfos, so we reset it in place instead:
- segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
- }
- rollbackSegmentInfos = null;
- }
-
- hasChanges = rollbackHasChanges;
- }
-
- /// <summary>Release the write lock, if needed. </summary>
- ~DirectoryIndexReader()
- {
- try
- {
- if (writeLock != null)
- {
- writeLock.Release(); // release write lock
- writeLock = null;
- }
- }
- finally
+
+ /// <summary> IndexReader implementation that has access to a Directory.
+ /// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
+ /// "own" the directory, which means that they try to acquire a write lock
+ /// whenever index modifications are performed.
+ /// </summary>
+ abstract public class DirectoryIndexReader : IndexReader
+ {
+ protected internal Directory directory;
+ protected internal bool closeDirectory;
+ private IndexDeletionPolicy deletionPolicy;
+
+ private SegmentInfos segmentInfos;
+ private Lock writeLock;
+ private bool stale;
+ private readonly IDictionary<string, string> synced = new Dictionary<string, string>();
+
+ /// <summary>Used by commit() to record pre-commit state in case
+ /// rollback is necessary
+ /// </summary>
+ private bool rollbackHasChanges;
+ private SegmentInfos rollbackSegmentInfos;
+
+ protected internal bool readOnly;
+
+ internal virtual void Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory, bool readOnly)
+ {
+ this.directory = directory;
+ this.segmentInfos = segmentInfos;
+ this.closeDirectory = closeDirectory;
+ this.readOnly = readOnly;
+
+ if (!readOnly && segmentInfos != null)
+ {
+ // we assume that this segments_N was properly sync'd prior
+ for (int i = 0; i < segmentInfos.Count; i++)
+ {
+ SegmentInfo info = segmentInfos.Info(i);
+ IList<string> files = info.Files();
+ for (int j = 0; j < files.Count; j++)
+ synced[files[j]] = files[j];
+ }
+ }
+ }
+
+ protected internal DirectoryIndexReader()
+ {
+ }
+
+ internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory, bool readOnly)
+ : base()
+ {
+ Init(directory, segmentInfos, closeDirectory, readOnly);
+ }
+
+ internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
+ {
+ return Open(directory, closeDirectory, deletionPolicy, null, false);
+ }
+
+ internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy, IndexCommit commit, bool readOnly)
+ {
+ SegmentInfos.FindSegmentsFile finder = new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory, readOnly);
+
+ if (commit == null)
+ return (DirectoryIndexReader) finder.Run();
+ else
+ {
+ if (directory != commit.GetDirectory())
+ throw new System.IO.IOException("the specified commit does not match the specified Directory");
+ // this can and will directly throw IOException if the specified commit point has been deleted
+ return (DirectoryIndexReader)finder.DoBody(commit.GetSegmentsFileName());
+ }
+ }
+
+ private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
+ {
+ private bool closeDirectory;
+ private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
+ private bool readOnly;
+
+ internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1, bool readOnly)
+ : base(Param1)
+ {
+ this.closeDirectory = closeDirectory;
+ this.deletionPolicy = deletionPolicy;
+ this.readOnly = readOnly;
+ }
+
+ protected internal override object DoBody(System.String segmentFileName)
+ {
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(directory, segmentFileName);
+
+ DirectoryIndexReader reader;
+ if (infos.Count == 1)
+ {
+ // index is optimized
+ reader = SegmentReader.Get(readOnly, infos, infos.Info(0), closeDirectory);
+ }
+ else if (readOnly)
+ {
+ reader = new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory);
+ }
+ else
+ {
+ reader = new MultiSegmentReader(directory, infos, closeDirectory, false);
+ }
+ reader.SetDeletionPolicy(deletionPolicy);
+
+ return reader;
+ }
+ }
+
+ public override IndexReader Reopen()
+ {
+ lock (this)
+ {
+ EnsureOpen();
+
+ if (this.hasChanges || this.IsCurrent())
+ {
+ // this has changes, therefore we have the lock and don't need to reopen
+ // OR: the index in the directory hasn't changed - nothing to do here
+ return this;
+ }
+
+ return (DirectoryIndexReader)new AnonymousClassFindSegmentsFile1(this, directory).Run();
+ }
+ }
+
+ private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
+ {
+ private DirectoryIndexReader enclosingInstance;
+
+ public DirectoryIndexReader Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+
+ internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1)
+ : base(Param1)
{
- // {{Aroush-2.3.1}} do we need to call Finalize() here?
+ this.enclosingInstance = enclosingInstance;
}
- }
- }
+
+ protected internal override object DoBody(System.String segmentFileName)
+ {
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(directory, segmentFileName);
+
+ DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
+
+ if (Enclosing_Instance != newReader)
+ {
+ newReader.Init(directory, infos, Enclosing_Instance.closeDirectory, Enclosing_Instance.readOnly);
+ newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
+ }
+
+ return newReader;
+ }
+ }
+
+ /// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
+ protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
+
+ public virtual void SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
+ {
+ this.deletionPolicy = deletionPolicy;
+ }
+
+ /// <summary>Returns the directory this index resides in.</summary>
+ public override Directory Directory()
+ {
+ EnsureOpen();
+ return directory;
+ }
+
+ /// <summary> Version number when this IndexReader was opened.</summary>
+ public override long GetVersion()
+ {
+ EnsureOpen();
+ return segmentInfos.GetVersion();
+ }
+
+ /// <summary> Check whether this IndexReader is still using the
+ /// current (i.e., most recently committed) version of the
+ /// index. If a writer has committed any changes to the
+ /// index since this reader was opened, this will return
+ /// <code>false</code>, in which case you must open a new
+ /// IndexReader in order to see the changes. See the
+ /// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
+ /// flag which controls when the {@link IndexWriter}
+ /// actually commits changes to the index.
+ ///
+ /// </summary>
+ /// <throws> CorruptIndexException if the index is corrupt </throws>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ public override bool IsCurrent()
+ {
+ EnsureOpen();
+ return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
+ }
+
+ /// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
+ /// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
+ /// </returns>
+ public override bool IsOptimized()
+ {
+ EnsureOpen();
+ return segmentInfos.Count == 1 && HasDeletions() == false;
+ }
+
+ protected internal override void DoClose()
+ {
+ if (closeDirectory)
+ directory.Close();
+ }
+
+ /// <summary> Commit changes resulting from delete, undeleteAll, or
+ /// setNorm operations
+ ///
+ /// If an exception is hit, then either no changes or all
+ /// changes will have been committed to the index
+ /// (transactional semantics).
+ /// </summary>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ protected internal override void DoCommit()
+ {
+ if (hasChanges)
+ {
+ if (segmentInfos != null)
+ {
+
+ // Default deleter (for backwards compatibility) is
+ // KeepOnlyLastCommitDeleter:
+ IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
+
+ // Checkpoint the state we are about to change, in
+ // case we have to roll back:
+ StartCommit();
+
+ bool success = false;
+ try
+ {
+ CommitChanges();
+
+ // sync all the files we just wrote
+ for (int i = 0; i < segmentInfos.Count; i++)
+ {
+ SegmentInfo info = segmentInfos.Info(i);
+ IList<string> files = info.Files();
+ for (int j = 0; j < files.Count; j++)
+ {
+ string fileName = files[j];
+ if (!synced.ContainsKey(fileName))
+ {
+ System.Diagnostics.Debug.Assert(directory.FileExists(fileName));
+ directory.Sync(fileName);
+ synced[fileName] = fileName;
+ }
+ }
+ }
+
+ segmentInfos.Commit(directory);
+ success = true;
+ }
+ finally
+ {
+
+ if (!success)
+ {
+
+ // Rollback changes that were made to
+ // SegmentInfos but failed to get [fully]
+ // committed. This way this reader instance
+ // remains consistent (matched to what's
+ // actually in the index):
+ RollbackCommit();
+
+ // Recompute deletable files & remove them (so
+ // partially written .del files, etc, are
+ // removed):
+ deleter.Refresh();
+ }
+ }
+
+ // Have the deleter remove any now unreferenced
+ // files due to this commit:
+ deleter.Checkpoint(segmentInfos, true);
+
+ if (writeLock != null)
+ {
+ writeLock.Release(); // release write lock
+ writeLock = null;
+ }
+ }
+ else
+ CommitChanges();
+ }
+ hasChanges = false;
+ }
+
+ protected internal abstract void CommitChanges();
+
+ /// <summary> Tries to acquire the WriteLock on this directory.
+ /// this method is only valid if this IndexReader is directory owner.
+ ///
+ /// </summary>
+ /// <throws> StaleReaderException if the index has changed </throws>
+ /// <summary> since this reader was opened
+ /// </summary>
+ /// <throws> CorruptIndexException if the index is corrupt </throws>
+ /// <throws> LockObtainFailedException if another writer </throws>
+ /// <summary> has this index open (<code>write.lock</code> could not
+ /// be obtained)
+ /// </summary>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ protected internal override void AcquireWriteLock()
+ {
+ if (segmentInfos != null)
+ {
+ EnsureOpen();
+ if (stale)
+ throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+
+ if (this.writeLock == null)
+ {
+ Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
+ if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
+ // obtain write lock
+ {
+ throw new LockObtainFailedException("Index locked for write: " + writeLock);
+ }
+ this.writeLock = writeLock;
+
+ // we have to check whether index has changed since this reader was opened.
+ // if so, this reader is no longer valid for deletion
+ if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
+ {
+ stale = true;
+ this.writeLock.Release();
+ this.writeLock = null;
+ throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+ }
+ }
+ }
+ }
+
+ /// <summary> Should internally checkpoint state that will change
+ /// during commit so that we can rollback if necessary.
+ /// </summary>
+ internal virtual void StartCommit()
+ {
+ if (segmentInfos != null)
+ {
+ rollbackSegmentInfos = (SegmentInfos)segmentInfos.Clone();
+ }
+ rollbackHasChanges = hasChanges;
+ }
+
+ /// <summary> Rolls back state to just before the commit (this is
+ /// called by commit() if there is some exception while
+ /// committing).
+ /// </summary>
+ internal virtual void RollbackCommit()
+ {
+ if (segmentInfos != null)
+ {
+ for (int i = 0; i < segmentInfos.Count; i++)
+ {
+ // Rollback each segmentInfo. Because the
+ // SegmentReader holds a reference to the
+ // SegmentInfo we can't [easily] just replace
+ // segmentInfos, so we reset it in place instead:
+ segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
+ }
+ rollbackSegmentInfos = null;
+ }
+
+ hasChanges = rollbackHasChanges;
+ }
+
+ /// <summary>Release the write lock, if needed. </summary>
+ ~DirectoryIndexReader()
+ {
+ try
+ {
+ if (writeLock != null)
+ {
+ writeLock.Release(); // release write lock
+ writeLock = null;
+ }
+ }
+ finally
+ {
+ // {{Aroush-2.3.1}} do we need to call Finalize() here?
+ }
+ }
+
+ private class ReaderCommit : IndexCommit
+ {
+ private string segmentsFileName;
+ internal ICollection<string> files;
+ internal Directory dir;
+ internal long generation;
+ internal long version;
+ internal readonly bool isOptimized;
+
+ internal ReaderCommit(SegmentInfos infos, Directory dir)
+ {
+ segmentsFileName = infos.GetCurrentSegmentFileName();
+ this.dir = dir;
+ int size = infos.Count;
+ files = new List<string>(size);
+ files.Add(segmentsFileName);
+ for (int i = 0; i < size; i++)
+ {
+ SegmentInfo info = infos.Info(i);
+ if (info.dir == dir)
+ SupportClass.CollectionsSupport.AddAll(info.Files(), files);
+ }
+ version = infos.GetVersion();
+ generation = infos.GetGeneration();
+ isOptimized = infos.Count == 1 && !infos.Info(0).HasDeletions();
+ }
+
+ public override bool IsOptimized()
+ {
+ return isOptimized;
+ }
+ public override string GetSegmentsFileName()
+ {
+ return segmentsFileName;
+ }
+ public override ICollection<string> GetFileNames()
+ {
+ return files;
+ }
+ public override Directory GetDirectory()
+ {
+ return dir;
+ }
+ public override long GetVersion()
+ {
+ return version;
+ }
+ public override long GetGeneration()
+ {
+ return generation;
+ }
+ public override bool IsDeleted()
+ {
+ return false;
+ }
+ }
+
+ /**
+ * Expert: return the IndexCommit that this reader has
+ * opened.
+ *
+ * <p><b>WARNING</b>: this API is new and experimental and
+ * may suddenly change.</p>
+ */
+ public override IndexCommit GetIndexCommit()
+ {
+ return new ReaderCommit(segmentInfos, directory);
+ }
+
+ /** @see IndexReader#listCommits */
+ public new static ICollection<IndexCommitPoint> ListCommits(Directory dir)
+ {
+
+ string[] files = dir.List();
+ if (files == null)
+ throw new System.IO.IOException("cannot read directory " + dir + ": list() returned null");
+
+ ICollection<IndexCommitPoint> commits = new List<IndexCommitPoint>();
+
+ SegmentInfos latest = new SegmentInfos();
+ latest.Read(dir);
+ long currentGen = latest.GetGeneration();
+
+ commits.Add(new ReaderCommit(latest, dir));
+
+ for (int i = 0; i < files.Length; i++)
+ {
+
+ String fileName = files[i];
+
+ if (fileName.StartsWith(IndexFileNames.SEGMENTS) &&
+ !fileName.Equals(IndexFileNames.SEGMENTS_GEN) &&
+ SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen)
+ {
+ SegmentInfos sis = new SegmentInfos();
+ try
+ {
+ // IOException allowed to throw there, in case
+ // segments_N is corrupt
+ sis.Read(dir, fileName);
+ }
+ catch (System.Exception)
+ {
+ // LUCENE-948: on NFS (and maybe others), if
+ // you have writers switching back and forth
+ // between machines, it's very likely that the
+ // dir listing will be stale and will claim a
+ // file segments_X exists when in fact it
+ // doesn't. So, we catch this and handle it
+ // as if the file does not exist
+ sis = null;
+ }
+
+ if (sis != null)
+ commits.Add(new ReaderCommit(sis, dir));
+ }
+ }
+
+ return commits;
+ }
+ }
}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocConsumer.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocConsumerPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumer.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumerPerField.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumers.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumersPerField.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Fieldable = Lucene.Net.Documents.Fieldable;
+
+namespace Lucene.Net.Index
+{
+ internal sealed class DocFieldConsumersPerField : DocFieldConsumerPerField
+ {
+
+ internal readonly DocFieldConsumerPerField one;
+ internal readonly DocFieldConsumerPerField two;
+ internal readonly DocFieldConsumersPerThread perThread;
+
+ public DocFieldConsumersPerField(DocFieldConsumersPerThread perThread, DocFieldConsumerPerField one, DocFieldConsumerPerField two)
+ {
+ this.perThread = perThread;
+ this.one = one;
+ this.two = two;
+ }
+
+ internal override void processFields(Fieldable[] fields, int count)
+ {
+ one.processFields(fields, count);
+ two.processFields(fields, count);
+ }
+
+ internal override void abort()
+ {
+ try
+ {
+ one.abort();
+ }
+ finally
+ {
+ two.abort();
+ }
+ }
+ }
+}
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessor.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessorPerField.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverter.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverterPerField.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Fieldable = Lucene.Net.Documents.Fieldable;
+using Token = Lucene.Net.Analysis.Token;
+using TokenStream = Lucene.Net.Analysis.TokenStream;
+
+namespace Lucene.Net.Index
+{
+ /// <summary>
+ /// Holds state for inverting all occurrences of a single
+ /// field in the document. This class doesn't do anything
+ /// itself; instead, it forwards the tokens produced by
+ /// analysis to its own consumer
+ /// (InvertedDocConsumerPerField). It also interacts with an
+ /// endConsumer (InvertedDocEndConsumerPerField).
+ /// </summary>
+ internal sealed class DocInverterPerField : DocFieldConsumerPerField
+ {
+
+ private readonly DocInverterPerThread perThread;
+ private readonly FieldInfo fieldInfo;
+ internal readonly InvertedDocConsumerPerField consumer;
+ internal readonly InvertedDocEndConsumerPerField endConsumer;
+ internal readonly DocumentsWriter.DocState docState;
+ internal readonly DocInverter.FieldInvertState fieldState;
+
+ public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo)
+ {
+ this.perThread = perThread;
+ this.fieldInfo = fieldInfo;
+ docState = perThread.docState;
+ fieldState = perThread.fieldState;
+ this.consumer = perThread.consumer.addField(this, fieldInfo);
+ this.endConsumer = perThread.endConsumer.addField(this, fieldInfo);
+ }
+
+ internal override void abort()
+ {
+ consumer.abort();
+ endConsumer.abort();
+ }
+
+ internal override void processFields(Fieldable[] fields,
+ int count)
+ {
+
+ fieldState.reset(docState.doc.GetBoost());
+
+ int maxFieldLength = docState.maxFieldLength;
+
+ bool doInvert = consumer.start(fields, count);
+
+ for (int i = 0; i < count; i++)
+ {
+
+ Fieldable field = fields[i];
+
+ // TODO FI: this should be "genericized" to querying
+ // consumer if it wants to see this particular field
+ // tokenized.
+ if (field.IsIndexed() && doInvert)
+ {
+
+ if (fieldState.length > 0)
+ fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name);
+
+ if (!field.IsTokenized())
+ { // un-tokenized field
+ string stringValue = field.StringValue();
+ int valueLength = stringValue.Length;
+ Token token = perThread.localToken.Reinit(stringValue, fieldState.offset, fieldState.offset + valueLength);
+ bool success = false;
+ try
+ {
+ consumer.add(token);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ docState.docWriter.SetAborting();
+ }
+ fieldState.offset += valueLength;
+ fieldState.length++;
+ fieldState.position++;
+ }
+ else
+ { // tokenized field
+ TokenStream stream;
+ TokenStream streamValue = field.TokenStreamValue();
+
+ if (streamValue != null)
+ stream = streamValue;
+ else
+ {
+ // the field does not have a TokenStream,
+ // so we have to obtain one from the analyzer
+ System.IO.TextReader reader; // find or make Reader
+ System.IO.TextReader readerValue = field.ReaderValue();
+
+ if (readerValue != null)
+ reader = readerValue;
+ else
+ {
+ string stringValue = field.StringValue();
+ if (stringValue == null)
+ throw new System.ArgumentException("field must have either TokenStream, string or Reader value");
+ perThread.stringReader.Init(stringValue);
+ reader = perThread.stringReader;
+ }
+
+ // Tokenize field and add to postingTable
+ stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader);
+ }
+
+ // reset the TokenStream to the first token
+ stream.Reset();
+
+ try
+ {
+ int offsetEnd = fieldState.offset - 1;
+ Token localToken = perThread.localToken;
+ for (; ; )
+ {
+
+ // If we hit an exception in stream.next below
+ // (which is fairly common, eg if analyzer
+ // chokes on a given document), then it's
+ // non-aborting and (above) this one document
+ // will be marked as deleted, but still
+ // consume a docID
+ Token token = stream.Next(localToken);
+
+ if (token == null) break;
+ fieldState.position += (token.GetPositionIncrement() - 1);
+ bool success = false;
+ try
+ {
+ // If we hit an exception in here, we abort
+ // all buffered documents since the last
+ // flush, on the likelihood that the
+ // internal state of the consumer is now
+ // corrupt and should not be flushed to a
+ // new segment:
+ consumer.add(token);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ docState.docWriter.SetAborting();
+ }
+ fieldState.position++;
+ offsetEnd = fieldState.offset + token.EndOffset();
+
+ if (++fieldState.length >= maxFieldLength)
+ {
+ if (docState.infoStream != null)
+ docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens");
+ break;
+ }
+ }
+ fieldState.offset = offsetEnd + 1;
+ }
+ finally
+ {
+ stream.Close();
+ }
+ }
+
+ fieldState.boost *= field.GetBoost();
+ }
+ }
+
+ consumer.finish();
+ endConsumer.finish();
+ }
+ }
+}
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverterPerThread.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Token = Lucene.Net.Analysis.Token;
+
+namespace Lucene.Net.Index
+{
+ internal sealed class DocInverterPerThread : DocFieldConsumerPerThread
+ {
+ internal readonly DocInverter docInverter;
+ internal readonly InvertedDocConsumerPerThread consumer;
+ internal readonly InvertedDocEndConsumerPerThread endConsumer;
+ internal readonly Token localToken = new Token();
+ internal readonly DocumentsWriter.DocState docState;
+
+ internal readonly DocInverter.FieldInvertState fieldState = new DocInverter.FieldInvertState();
+
+ // Used to read a string value for a field
+ internal readonly ReusableStringReader stringReader = new ReusableStringReader();
+
+ public DocInverterPerThread(DocFieldProcessorPerThread docFieldProcessorPerThread, DocInverter docInverter)
+ {
+ this.docInverter = docInverter;
+ docState = docFieldProcessorPerThread.docState;
+ consumer = docInverter.consumer.addThread(this);
+ endConsumer = docInverter.endConsumer.addThread(this);
+ }
+
+ internal override void startDocument()
+ {
+ consumer.startDocument();
+ endConsumer.startDocument();
+ }
+
+ internal override DocumentsWriter.DocWriter finishDocument()
+ {
+ // TODO: allow endConsumer.finishDocument to also return
+ // a DocWriter
+ endConsumer.finishDocument();
+ return consumer.finishDocument();
+ }
+
+ internal override void abort()
+ {
+ try
+ {
+ consumer.abort();
+ }
+ finally
+ {
+ endConsumer.abort();
+ }
+ }
+
+ internal override DocFieldConsumerPerField addField(FieldInfo fi)
+ {
+ return new DocInverterPerField(this, fi);
+ }
+ }
+}