You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ar...@apache.org on 2008/06/25 04:52:24 UTC
svn commit: r671404 [1/10] -
/incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/
Author: aroush
Date: Tue Jun 24 19:52:22 2008
New Revision: 671404
URL: http://svn.apache.org/viewvc?rev=671404&view=rev
Log:
Release: Apache Lucene.Net.2.3.1 build 001 "Alpha"
Added:
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldSortedTermVectorMapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/KeepOnlyLastCommitDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogByteSizeMergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogDocMergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergeScheduler.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiSegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Payload.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/PositionBasedTermVectorMapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReade-2r.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SerialMergeScheduler.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SortedTermVectorMapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StaleReaderException.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntry.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntryFreqSortedComparator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorMapper.cs
Modified:
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfos.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldReaderException.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNameFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNames.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexModifier.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Package.html
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMergeQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermDocs.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermEnum.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositionVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositions.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Term.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermBuffer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermFreqVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermPositions.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorOffsetInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsWriter.cs
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CheckIndex.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,429 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Document = Lucene.Net.Documents.Document;
+using Directory = Lucene.Net.Store.Directory;
+using FSDirectory = Lucene.Net.Store.FSDirectory;
+using IndexInput = Lucene.Net.Store.IndexInput;
+
+namespace Lucene.Net.Index
+{
+
+ /// <summary> Basic tool to check the health of an index and write a
+ /// new segments file that removes reference to problematic
+ /// segments. There are many more checks that this tool
+ /// could do but does not yet, eg: reconstructing a segments
+ /// file by looking for all loadable segments (if no segments
+ /// file is found), removing specifically specified segments,
+ /// listing files that exist but are not referenced, etc.
+ /// </summary>
+
+ public class CheckIndex
+ {
+
+ public static System.IO.TextWriter out_Renamed;
+
+ private class MySegmentTermDocs : SegmentTermDocs
+ {
+
+ internal int delCount;
+
+ internal MySegmentTermDocs(SegmentReader p) : base(p)
+ {
+ }
+
+ public override void Seek(Term term)
+ {
+ base.Seek(term);
+ delCount = 0;
+ }
+
+ protected internal override void SkippingDoc()
+ {
+ delCount++;
+ }
+ }
+
+ /// <summary>Returns true if index is clean, else false.</summary>
+ public static bool Check(Directory dir, bool doFix)
+ {
+ System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
+ SegmentInfos sis = new SegmentInfos();
+
+ try
+ {
+ sis.Read(dir);
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("ERROR: could not read any segments file in directory");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ return false;
+ }
+
+ int numSegments = sis.Count;
+ System.String segmentsFileName = sis.GetCurrentSegmentFileName();
+ IndexInput input = null;
+ try
+ {
+ input = dir.OpenInput(segmentsFileName);
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("ERROR: could not open segments file in directory");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ return false;
+ }
+ int format = 0;
+ try
+ {
+ format = input.ReadInt();
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("ERROR: could not read segment file version in directory");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ return false;
+ }
+ finally
+ {
+ if (input != null)
+ input.Close();
+ }
+
+ System.String sFormat = "";
+ bool skip = false;
+
+ if (format == SegmentInfos.FORMAT)
+ sFormat = "FORMAT [Lucene Pre-2.1]";
+ if (format == SegmentInfos.FORMAT_LOCKLESS)
+ sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
+ else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
+ sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
+ else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
+ sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
+ else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE)
+ {
+ sFormat = "int=" + format + " [newer version of Lucene than this tool]";
+ skip = true;
+ }
+ else
+ {
+ sFormat = format + " [Lucene 1.3 or prior]";
+ }
+
+ out_Renamed.WriteLine("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
+
+ if (skip)
+ {
+ out_Renamed.WriteLine("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
+ return false;
+ }
+
+ SegmentInfos newSIS = (SegmentInfos) sis.Clone();
+ newSIS.Clear();
+ bool changed = false;
+ int totLoseDocCount = 0;
+ int numBadSegments = 0;
+ for (int i = 0; i < numSegments; i++)
+ {
+ SegmentInfo info = sis.Info(i);
+ out_Renamed.WriteLine(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
+ int toLoseDocCount = info.docCount;
+
+ SegmentReader reader = null;
+
+ try
+ {
+ out_Renamed.WriteLine(" compound=" + info.GetUseCompoundFile());
+ out_Renamed.WriteLine(" numFiles=" + info.Files().Count);
+ out_Renamed.WriteLine(String.Format(nf, " size (MB)={0:f}", new Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
+ int docStoreOffset = info.GetDocStoreOffset();
+ if (docStoreOffset != - 1)
+ {
+ out_Renamed.WriteLine(" docStoreOffset=" + docStoreOffset);
+ out_Renamed.WriteLine(" docStoreSegment=" + info.GetDocStoreSegment());
+ out_Renamed.WriteLine(" docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
+ }
+ System.String delFileName = info.GetDelFileName();
+ if (delFileName == null)
+ out_Renamed.WriteLine(" no deletions");
+ else
+ out_Renamed.WriteLine(" has deletions [delFileName=" + delFileName + "]");
+ out_Renamed.Write(" test: open reader.........");
+ reader = SegmentReader.Get(info);
+ int numDocs = reader.NumDocs();
+ toLoseDocCount = numDocs;
+ if (reader.HasDeletions())
+ out_Renamed.WriteLine("OK [" + (info.docCount - numDocs) + " deleted docs]");
+ else
+ out_Renamed.WriteLine("OK");
+
+ out_Renamed.Write(" test: fields, norms.......");
+ System.Collections.ICollection fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
+ System.Collections.IEnumerator it = fieldNames.GetEnumerator();
+ while (it.MoveNext())
+ {
+ System.String fieldName = (System.String) it.Current;
+ byte[] b = reader.Norms(fieldName);
+ if (b.Length != info.docCount)
+ throw new System.SystemException("norms for field \"" + fieldName + "\" is length " + b.Length + " != maxDoc " + info.docCount);
+ }
+ out_Renamed.WriteLine("OK [" + fieldNames.Count + " fields]");
+
+ out_Renamed.Write(" test: terms, freq, prox...");
+ TermEnum termEnum = reader.Terms();
+ TermPositions termPositions = reader.TermPositions();
+
+ // Used only to count up # deleted docs for this
+ // term
+ MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
+
+ long termCount = 0;
+ long totFreq = 0;
+ long totPos = 0;
+ while (termEnum.Next())
+ {
+ termCount++;
+ Term term = termEnum.Term();
+ int docFreq = termEnum.DocFreq();
+ termPositions.Seek(term);
+ int lastDoc = - 1;
+ int freq0 = 0;
+ totFreq += docFreq;
+ while (termPositions.Next())
+ {
+ freq0++;
+ int doc = termPositions.Doc();
+ int freq = termPositions.Freq();
+ if (doc <= lastDoc)
+ {
+ throw new System.SystemException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
+ }
+ lastDoc = doc;
+ if (freq <= 0)
+ {
+ throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+ }
+
+ int lastPos = - 1;
+ totPos += freq;
+ for (int j = 0; j < freq; j++)
+ {
+ int pos = termPositions.NextPosition();
+ if (pos < 0)
+ {
+ throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
+ }
+ if (pos <= lastPos)
+ {
+ throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
+ }
+ }
+ }
+
+ // Now count how many deleted docs occurred in
+ // this term:
+ int delCount;
+ if (reader.HasDeletions())
+ {
+ myTermDocs.Seek(term);
+ while (myTermDocs.Next())
+ {
+ }
+ delCount = myTermDocs.delCount;
+ }
+ else
+ delCount = 0;
+
+ if (freq0 + delCount != docFreq)
+ {
+ throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
+ }
+ }
+
+ out_Renamed.WriteLine("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
+
+ out_Renamed.Write(" test: stored fields.......");
+ int docCount = 0;
+ long totFields = 0;
+ for (int j = 0; j < info.docCount; j++)
+ if (!reader.IsDeleted(j))
+ {
+ docCount++;
+ Document doc = reader.Document(j);
+ totFields += doc.GetFields().Count;
+ }
+
+ if (docCount != reader.NumDocs())
+ throw new System.SystemException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
+
+ out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total field count; avg {1:f} fields per doc]", new Object[] { totFields, (((float)totFields) / docCount) }));
+
+ out_Renamed.Write(" test: term vectors........");
+ int totVectors = 0;
+ for (int j = 0; j < info.docCount; j++)
+ if (!reader.IsDeleted(j))
+ {
+ TermFreqVector[] tfv = reader.GetTermFreqVectors(j);
+ if (tfv != null)
+ totVectors += tfv.Length;
+ }
+
+ out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new Object[] { totVectors, (((float)totVectors) / docCount) }));
+ out_Renamed.WriteLine("");
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("FAILED");
+ System.String comment;
+ if (doFix)
+ comment = "will remove reference to this segment (-fix is specified)";
+ else
+ comment = "would remove reference to this segment (-fix was not specified)";
+ out_Renamed.WriteLine(" WARNING: " + comment + "; full exception:");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ out_Renamed.WriteLine("");
+ totLoseDocCount += toLoseDocCount;
+ numBadSegments++;
+ changed = true;
+ continue;
+ }
+ finally
+ {
+ if (reader != null)
+ reader.Close();
+ }
+
+ // Keeper
+ newSIS.Add(info.Clone());
+ }
+
+ if (!changed)
+ {
+ out_Renamed.WriteLine("No problems were detected with this index.\n");
+ return true;
+ }
+ else
+ {
+ out_Renamed.WriteLine("WARNING: " + numBadSegments + " broken segments detected");
+ if (doFix)
+ out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents will be lost");
+ else
+ out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
+ out_Renamed.WriteLine();
+ }
+
+ if (doFix)
+ {
+ out_Renamed.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
+ for (int i = 0; i < 5; i++)
+ {
+ try
+ {
+ System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
+ }
+ catch (System.Threading.ThreadInterruptedException)
+ {
+ SupportClass.ThreadClass.Current().Interrupt();
+ i--;
+ continue;
+ }
+
+ out_Renamed.WriteLine(" " + (5 - i) + "...");
+ }
+ out_Renamed.Write("Writing...");
+ try
+ {
+ newSIS.Write(dir);
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("FAILED; exiting");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ return false;
+ }
+ out_Renamed.WriteLine("OK");
+ out_Renamed.WriteLine("Wrote new segments file \"" + newSIS.GetCurrentSegmentFileName() + "\"");
+ }
+ else
+ {
+ out_Renamed.WriteLine("NOTE: would write new segments file [-fix was not specified]");
+ }
+ out_Renamed.WriteLine("");
+
+ return false;
+ }
+
+ [STAThread]
+ public static void Main(System.String[] args)
+ {
+
+ bool doFix = false;
+ for (int i = 0; i < args.Length; i++)
+ if (args[i].Equals("-fix"))
+ {
+ doFix = true;
+ break;
+ }
+
+ if (args.Length != (doFix ? 2 : 1))
+ {
+ out_Renamed.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has has any\n" + "corrup
tion, else 0.\n");
+ System.Environment.Exit(1);
+ }
+
+ System.String dirName = args[0];
+ out_Renamed.WriteLine("\nOpening index @ " + dirName + "\n");
+ Directory dir = null;
+ try
+ {
+ dir = FSDirectory.GetDirectory(dirName);
+ }
+ catch (System.Exception t)
+ {
+ out_Renamed.WriteLine("ERROR: could not open directory \"" + dirName + "\"; exiting");
+ out_Renamed.Write(t.StackTrace);
+ out_Renamed.Flush();
+ System.Environment.Exit(1);
+ }
+
+ bool isClean = Check(dir, doFix);
+
+ int exitCode;
+ if (isClean)
+ exitCode = 0;
+ else
+ exitCode = 1;
+ System.Environment.Exit(exitCode);
+ }
+ static CheckIndex()
+ {
+ System.IO.StreamWriter temp_writer;
+ temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
+ temp_writer.AutoFlush = true;
+ out_Renamed = temp_writer;
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CompoundFileReader.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs Tue Jun 24 19:52:22 2008
@@ -16,9 +16,10 @@
*/
using System;
+
+using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
using Directory = Lucene.Net.Store.Directory;
using IndexInput = Lucene.Net.Store.IndexInput;
-using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
using IndexOutput = Lucene.Net.Store.IndexOutput;
using Lock = Lucene.Net.Store.Lock;
@@ -30,14 +31,15 @@
/// This class implements a directory, but is limited to only read operations.
/// Directory methods that would normally modify data throw an exception.
///
+ ///
/// </summary>
- /// <author> Dmitry Serebrennikov
- /// </author>
- /// <version> $Id: CompoundFileReader.java 472959 2006-11-09 16:21:50Z yonik $
+ /// <version> $Id: CompoundFileReader.java 564236 2007-08-09 15:21:19Z gsingers $
/// </version>
public class CompoundFileReader : Directory
{
+ private int readBufferSize;
+
private sealed class FileEntry
{
internal long offset;
@@ -53,16 +55,21 @@
private System.Collections.Hashtable entries = new System.Collections.Hashtable();
- public CompoundFileReader(Directory dir, System.String name)
+ public CompoundFileReader(Directory dir, System.String name):this(dir, name, BufferedIndexInput.BUFFER_SIZE)
+ {
+ }
+
+ public CompoundFileReader(Directory dir, System.String name, int readBufferSize)
{
directory = dir;
fileName = name;
+ this.readBufferSize = readBufferSize;
bool success = false;
try
{
- stream = dir.OpenInput(name);
+ stream = dir.OpenInput(name, readBufferSize);
// read the directory and init files
int count = stream.ReadVInt();
@@ -133,14 +140,23 @@
{
lock (this)
{
+ // Default to readBufferSize passed in when we were opened
+ return OpenInput(id, readBufferSize);
+ }
+ }
+
+ public override IndexInput OpenInput(System.String id, int readBufferSize)
+ {
+ lock (this)
+ {
if (stream == null)
throw new System.IO.IOException("Stream closed");
FileEntry entry = (FileEntry) entries[id];
if (entry == null)
throw new System.IO.IOException("No sub-file with id " + id + " found");
-
- return new CSIndexInput(stream, entry.offset, entry.length);
+
+ return new CSIndexInput(stream, entry.offset, entry.length, readBufferSize);
}
}
@@ -220,7 +236,11 @@
internal long fileOffset;
internal long length;
- internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length)
+ internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length) : this(base_Renamed, fileOffset, length, BufferedIndexInput.BUFFER_SIZE)
+ {
+ }
+
+ internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length, int readBufferSize) : base(readBufferSize)
{
this.base_Renamed = base_Renamed;
this.fileOffset = fileOffset;
@@ -236,7 +256,7 @@
/// </param>
/// <param name="len">the number of bytes to read
/// </param>
- public override void ReadInternal(byte[] b, int offset, int len)
+ protected internal override void ReadInternal(byte[] b, int offset, int len)
{
lock (base_Renamed)
{
@@ -244,16 +264,16 @@
if (start + len > length)
throw new System.IO.IOException("read past EOF");
base_Renamed.Seek(fileOffset + start);
- base_Renamed.ReadBytes(b, offset, len);
+ base_Renamed.ReadBytes(b, offset, len, false);
}
}
/// <summary>Expert: implements seek. Sets current position in this file, where
/// the next {@link #ReadInternal(byte[],int,int)} will occur.
/// </summary>
- /// <seealso cref="#ReadInternal(byte[],int,int)">
+ /// <seealso cref="ReadInternal(byte[],int,int)">
/// </seealso>
- public override void SeekInternal(long pos)
+ protected internal override void SeekInternal(long pos)
{
}
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CompoundFileWriter.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs Tue Jun 24 19:52:22 2008
@@ -16,9 +16,10 @@
*/
using System;
+
using Directory = Lucene.Net.Store.Directory;
-using IndexOutput = Lucene.Net.Store.IndexOutput;
using IndexInput = Lucene.Net.Store.IndexInput;
+using IndexOutput = Lucene.Net.Store.IndexOutput;
namespace Lucene.Net.Index
{
@@ -43,10 +44,9 @@
/// contains a long pointer to the start of this file's data section, and a String
/// with that file's name.
///
+ ///
/// </summary>
- /// <author> Dmitry Serebrennikov
- /// </author>
- /// <version> $Id: CompoundFileWriter.java 472959 2006-11-09 16:21:50Z yonik $
+ /// <version> $Id: CompoundFileWriter.java 606441 2007-12-22 10:06:28Z mikemccand $
/// </version>
public sealed class CompoundFileWriter
{
@@ -69,19 +69,23 @@
private System.Collections.Hashtable ids;
private System.Collections.ArrayList entries;
private bool merged = false;
-
+ private SegmentMerger.CheckAbort checkAbort;
/// <summary>Create the compound stream in the specified file. The file name is the
/// entire name (no extensions are added).
/// </summary>
/// <throws> NullPointerException if <code>dir</code> or <code>name</code> is null </throws>
- public CompoundFileWriter(Directory dir, System.String name)
+ public CompoundFileWriter(Directory dir, System.String name) : this(dir, name, null)
+ {
+ }
+
+ internal CompoundFileWriter(Directory dir, System.String name, SegmentMerger.CheckAbort checkAbort)
{
if (dir == null)
throw new System.NullReferenceException("directory cannot be null");
if (name == null)
throw new System.NullReferenceException("name cannot be null");
-
+ this.checkAbort = checkAbort;
directory = dir;
fileName = name;
ids = new System.Collections.Hashtable();
@@ -172,7 +176,7 @@
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
- byte[] buffer = new byte[1024];
+ byte[] buffer = new byte[16384];
it = entries.GetEnumerator();
while (it.MoveNext())
{
@@ -233,6 +237,10 @@
is_Renamed.ReadBytes(buffer, 0, len);
os.WriteBytes(buffer, len);
remainder -= len;
+ if (checkAbort != null)
+ // Roughly every 2 MB we will check if
+ // it's time to abort
+ checkAbort.Work(80);
}
// Verify that remainder is 0
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Directory = Lucene.Net.Store.Directory;
+
+namespace Lucene.Net.Index
+{
+
+ /// <summary>A {@link MergeScheduler} that runs each merge using a
+ /// separate thread, up until a maximum number of threads
+ /// ({@link #setMaxThreadCount}) at which points merges are
+ /// run in the foreground, serially. This is a simple way
+ /// to use concurrency in the indexing process without
+ /// having to create and manage application level
+ /// threads.
+ /// </summary>
+
+ public class ConcurrentMergeScheduler : MergeScheduler
+ {
+
+ private int mergeThreadPriority = - 1;
+
+ private System.Collections.IList mergeThreads = new System.Collections.ArrayList();
+ private int maxThreadCount = 3;
+
+ private System.Collections.IList exceptions = new System.Collections.ArrayList();
+ private Directory dir;
+
+ private bool closed;
+ private IndexWriter writer;
+
+ public ConcurrentMergeScheduler()
+ {
+ if (allInstances != null)
+ {
+ // Only for testing
+ AddMyself();
+ }
+ }
+
+ /// <summary>Sets the max # simultaneous threads that may be
+ /// running. If a merge is necessary yet we already have
+ /// this many threads running, the merge is returned back
+ /// to IndexWriter so that it runs in the "foreground".
+ /// </summary>
+ public virtual void SetMaxThreadCount(int count)
+ {
+ if (count < 1)
+ throw new System.ArgumentException("count should be at least 1");
+ maxThreadCount = count;
+ }
+
+ /// <summary>Get the max # simultaneous threads that may be</summary>
+ /// <seealso cref="setMaxThreadCount.">
+ /// </seealso>
+ public virtual int GetMaxThreadCount()
+ {
+ return maxThreadCount;
+ }
+
+ /// <summary>Return the priority that merge threads run at. By
+ /// default the priority is 1 plus the priority of (ie,
+ /// slightly higher priority than) the first thread that
+ /// calls merge.
+ /// </summary>
+ public virtual int GetMergeThreadPriority()
+ {
+ lock (this)
+ {
+ InitMergeThreadPriority();
+ return mergeThreadPriority;
+ }
+ }
+
+ /// <summary>Return the priority that merge threads run at. </summary>
+ public virtual void SetMergeThreadPriority(int pri)
+ {
+ lock (this)
+ {
+ if (pri > (int) System.Threading.ThreadPriority.Highest || pri < (int) System.Threading.ThreadPriority.Lowest)
+ throw new System.ArgumentException("priority must be in range " + (int) System.Threading.ThreadPriority.Lowest + " .. " + (int) System.Threading.ThreadPriority.Highest + " inclusive");
+ mergeThreadPriority = pri;
+
+ int numThreads = MergeThreadCount();
+ for (int i = 0; i < numThreads; i++)
+ {
+ MergeThread merge = (MergeThread) mergeThreads[i];
+ merge.SetThreadPriority(pri);
+ }
+ }
+ }
+
+ private void Message(System.String message)
+ {
+ if (writer != null)
+ writer.Message("CMS: " + message);
+ }
+
+ private void InitMergeThreadPriority()
+ {
+ lock (this)
+ {
+ if (mergeThreadPriority == - 1)
+ {
+ // Default to slightly higher priority than our
+ // calling thread
+ mergeThreadPriority = 1 + (System.Int32) SupportClass.ThreadClass.Current().Priority;
+ if (mergeThreadPriority > (int) System.Threading.ThreadPriority.Highest)
+ mergeThreadPriority = (int) System.Threading.ThreadPriority.Highest;
+ }
+ }
+ }
+
+ public override void Close()
+ {
+ closed = true;
+ }
+
+ public virtual void Sync()
+ {
+ lock (this)
+ {
+ while (MergeThreadCount() > 0)
+ {
+ Message("now wait for threads; currently " + mergeThreads.Count + " still running");
+ int count = mergeThreads.Count;
+ for (int i = 0; i < count; i++)
+ Message(" " + i + ": " + ((MergeThread) mergeThreads[i]));
+
+ try
+ {
+ System.Threading.Monitor.Wait(this);
+ }
+ catch (System.Threading.ThreadInterruptedException)
+ {
+ }
+ }
+ }
+ }
+
+ private int MergeThreadCount()
+ {
+ lock (this)
+ {
+ int count = 0;
+ int numThreads = mergeThreads.Count;
+ for (int i = 0; i < numThreads; i++)
+ if (((MergeThread) mergeThreads[i]).IsAlive)
+ count++;
+ return count;
+ }
+ }
+
+ public override void Merge(IndexWriter writer)
+ {
+
+ this.writer = writer;
+
+ InitMergeThreadPriority();
+
+ dir = writer.GetDirectory();
+
+ // First, quickly run through the newly proposed merges
+ // and add any orthogonal merges (ie a merge not
+ // involving segments already pending to be merged) to
+ // the queue. If we are way behind on merging, many of
+ // these newly proposed merges will likely already be
+ // registered.
+
+ Message("now merge");
+ Message(" index: " + writer.SegString());
+
+ // Iterate, pulling from the IndexWriter's queue of
+ // pending merges, until its empty:
+ while (true)
+ {
+
+ // TODO: we could be careful about which merges to do in
+ // the BG (eg maybe the "biggest" ones) vs FG, which
+ // merges to do first (the easiest ones?), etc.
+
+ MergePolicy.OneMerge merge = writer.GetNextMerge();
+ if (merge == null)
+ {
+ Message(" no more merges pending; now return");
+ return ;
+ }
+
+ // We do this w/ the primary thread to keep
+ // deterministic assignment of segment names
+ writer.MergeInit(merge);
+
+ Message(" consider merge " + merge.SegString(dir));
+
+ if (merge.isExternal)
+ {
+ Message(" merge involves segments from an external directory; now run in foreground");
+ }
+ else
+ {
+ lock (this)
+ {
+ if (MergeThreadCount() < maxThreadCount)
+ {
+ // OK to spawn a new merge thread to handle this
+ // merge:
+ MergeThread merger = new MergeThread(this, writer, merge);
+ mergeThreads.Add(merger);
+ Message(" launch new thread [" + merger.Name + "]");
+ merger.SetThreadPriority(mergeThreadPriority);
+ merger.IsBackground = true;
+ merger.Start();
+ continue;
+ }
+ else
+ Message(" too many merge threads running; run merge in foreground");
+ }
+ }
+
+ // Too many merge threads already running, so we do
+ // this in the foreground of the calling thread
+ writer.Merge(merge);
+ }
+ }
+
+ private class MergeThread:SupportClass.ThreadClass
+ {
+ private void InitBlock(ConcurrentMergeScheduler enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private ConcurrentMergeScheduler enclosingInstance;
+ public ConcurrentMergeScheduler Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+
+ internal IndexWriter writer;
+ internal MergePolicy.OneMerge startMerge;
+ internal MergePolicy.OneMerge runningMerge;
+
+ public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
+ {
+ InitBlock(enclosingInstance);
+ this.writer = writer;
+ this.startMerge = startMerge;
+ }
+
+ public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
+ {
+ lock (this)
+ {
+ runningMerge = merge;
+ }
+ }
+
+ public virtual MergePolicy.OneMerge GetRunningMerge()
+ {
+ lock (this)
+ {
+ return runningMerge;
+ }
+ }
+
+ public virtual void SetThreadPriority(int pri)
+ {
+ try
+ {
+ Priority = (System.Threading.ThreadPriority) pri;
+ }
+ catch (System.NullReferenceException)
+ {
+ // Strangely, Sun's JDK 1.5 on Linux sometimes
+ // throws NPE out of here...
+ }
+ catch (System.Security.SecurityException)
+ {
+ // Ignore this because we will still run fine with
+ // normal thread priority
+ }
+ }
+
+ override public void Run()
+ {
+
+ // First time through the while loop we do the merge
+ // that we were started with:
+ MergePolicy.OneMerge merge = this.startMerge;
+
+ try
+ {
+
+ Enclosing_Instance.Message(" merge thread: start");
+
+ while (true)
+ {
+ SetRunningMerge(merge);
+ writer.Merge(merge);
+
+ // Subsequent times through the loop we do any new
+ // merge that writer says is necessary:
+ merge = writer.GetNextMerge();
+ if (merge != null)
+ {
+ writer.MergeInit(merge);
+ Enclosing_Instance.Message(" merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
+ }
+ else
+ break;
+ }
+
+ Enclosing_Instance.Message(" merge thread: done");
+ }
+ catch (System.Exception exc)
+ {
+
+ if (merge != null)
+ {
+ merge.SetException(exc);
+ writer.AddMergeException(merge);
+ }
+
+ // Ignore the exception if it was due to abort:
+ if (!(exc is MergePolicy.MergeAbortedException))
+ {
+ lock (Enclosing_Instance)
+ {
+ Enclosing_Instance.exceptions.Add(exc);
+ }
+
+ if (!Enclosing_Instance.suppressExceptions)
+ {
+ // suppressExceptions is normally only set during
+ // testing.
+ Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
+ throw new MergePolicy.MergeException(exc);
+ }
+ }
+ }
+ finally
+ {
+ lock (Enclosing_Instance)
+ {
+ Enclosing_Instance.mergeThreads.Remove(this);
+ System.Threading.Monitor.PulseAll(Enclosing_Instance);
+ }
+ }
+ }
+
+ public override System.String ToString()
+ {
+ MergePolicy.OneMerge merge = GetRunningMerge();
+ if (merge == null)
+ merge = startMerge;
+ return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
+ }
+ }
+
+ internal static bool anyExceptions = false;
+
+ /// <summary>Used for testing </summary>
+ public static bool AnyUnhandledExceptions()
+ {
+ lock (allInstances.SyncRoot)
+ {
+ int count = allInstances.Count;
+ // Make sure all outstanding threads are done so we see
+ // any exceptions they may produce:
+ for (int i = 0; i < count; i++)
+ ((ConcurrentMergeScheduler) allInstances[i]).Sync();
+ return anyExceptions;
+ }
+ }
+
+ /// <summary>Used for testing </summary>
+ private void AddMyself()
+ {
+ lock (allInstances.SyncRoot)
+ {
+ int size = 0;
+ int upto = 0;
+ for (int i = 0; i < size; i++)
+ {
+ ConcurrentMergeScheduler other = (ConcurrentMergeScheduler) allInstances[i];
+ if (!(other.closed && 0 == other.MergeThreadCount()))
+ // Keep this one for now: it still has threads or
+ // may spawn new threads
+ allInstances[upto++] = other;
+ }
+ ((System.Collections.IList) ((System.Collections.ArrayList) allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
+ allInstances.Add(this);
+ }
+ }
+
+ private bool suppressExceptions;
+
+ /// <summary>Used for testing </summary>
+ internal virtual void SetSuppressExceptions()
+ {
+ suppressExceptions = true;
+ }
+
+ /// <summary>Used for testing </summary>
+ internal virtual void ClearSuppressExceptions()
+ {
+ suppressExceptions = false;
+ }
+
+ /// <summary>Used for testing </summary>
+ private static System.Collections.IList allInstances;
+ public static void SetTestMode()
+ {
+ allInstances = new System.Collections.ArrayList();
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CorruptIndexException.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Lucene.Net.Index
+{
+
+ /// <summary> This exception is thrown when Lucene detects
+ /// an inconsistency in the index.
+ /// </summary>
+ [Serializable]
+ public class CorruptIndexException : System.IO.IOException
+ {
+ public CorruptIndexException(System.String message) : base(message)
+ {
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListReader.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using IndexInput = Lucene.Net.Store.IndexInput;
+
+namespace Lucene.Net.Index
+{
+
+ /// <summary> Implements the skip list reader for the default posting list format
+ /// that stores positions and payloads.
+ ///
+ /// </summary>
+ class DefaultSkipListReader : MultiLevelSkipListReader
+ {
+ private bool currentFieldStoresPayloads;
+ private long[] freqPointer;
+ private long[] proxPointer;
+ private int[] payloadLength;
+
+ private long lastFreqPointer;
+ private long lastProxPointer;
+ private int lastPayloadLength;
+
+
+ internal DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) : base(skipStream, maxSkipLevels, skipInterval)
+ {
+ freqPointer = new long[maxSkipLevels];
+ proxPointer = new long[maxSkipLevels];
+ payloadLength = new int[maxSkipLevels];
+ }
+
+ internal virtual void Init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, bool storesPayloads)
+ {
+ base.Init(skipPointer, df);
+ this.currentFieldStoresPayloads = storesPayloads;
+ lastFreqPointer = freqBasePointer;
+ lastProxPointer = proxBasePointer;
+
+ for (int i = 0; i < freqPointer.Length; i++) freqPointer[i] = freqBasePointer;
+ for (int i = 0; i < proxPointer.Length; i++) proxPointer[i] = proxBasePointer;
+ for (int i = 0; i < payloadLength.Length; i++) payloadLength[i] = 0;
+ }
+
+ /// <summary>Returns the freq pointer of the doc to which the last call of
+ /// {@link MultiLevelSkipListReader#SkipTo(int)} has skipped.
+ /// </summary>
+ internal virtual long GetFreqPointer()
+ {
+ return lastFreqPointer;
+ }
+
+ /// <summary>Returns the prox pointer of the doc to which the last call of
+ /// {@link MultiLevelSkipListReader#SkipTo(int)} has skipped.
+ /// </summary>
+ internal virtual long GetProxPointer()
+ {
+ return lastProxPointer;
+ }
+
+ /// <summary>Returns the payload length of the payload stored just before
+ /// the doc to which the last call of {@link MultiLevelSkipListReader#SkipTo(int)}
+ /// has skipped.
+ /// </summary>
+ internal virtual int GetPayloadLength()
+ {
+ return lastPayloadLength;
+ }
+
+ protected internal override void SeekChild(int level)
+ {
+ base.SeekChild(level);
+ freqPointer[level] = lastFreqPointer;
+ proxPointer[level] = lastProxPointer;
+ payloadLength[level] = lastPayloadLength;
+ }
+
+ protected internal override void SetLastSkipData(int level)
+ {
+ base.SetLastSkipData(level);
+ lastFreqPointer = freqPointer[level];
+ lastProxPointer = proxPointer[level];
+ lastPayloadLength = payloadLength[level];
+ }
+
+
+ protected internal override int ReadSkipData(int level, IndexInput skipStream)
+ {
+ int delta;
+ if (currentFieldStoresPayloads)
+ {
+ // the current field stores payloads.
+ // if the doc delta is odd then we have
+ // to read the current payload length
+ // because it differs from the length of the
+ // previous payload
+ delta = skipStream.ReadVInt();
+ if ((delta & 1) != 0)
+ {
+ payloadLength[level] = skipStream.ReadVInt();
+ }
+ delta = SupportClass.Number.URShift(delta, 1);
+ }
+ else
+ {
+ delta = skipStream.ReadVInt();
+ }
+ freqPointer[level] += skipStream.ReadVInt();
+ proxPointer[level] += skipStream.ReadVInt();
+
+ return delta;
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListWriter.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using IndexOutput = Lucene.Net.Store.IndexOutput;
+
+namespace Lucene.Net.Index
+{
+
+
+ /// <summary> Implements the skip list writer for the default posting list format
+ /// that stores positions and payloads.
+ ///
+ /// </summary>
+ class DefaultSkipListWriter : MultiLevelSkipListWriter
+ {
+ private int[] lastSkipDoc;
+ private int[] lastSkipPayloadLength;
+ private long[] lastSkipFreqPointer;
+ private long[] lastSkipProxPointer;
+
+ private IndexOutput freqOutput;
+ private IndexOutput proxOutput;
+
+ private int curDoc;
+ private bool curStorePayloads;
+ private int curPayloadLength;
+ private long curFreqPointer;
+ private long curProxPointer;
+
+ internal DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) : base(skipInterval, numberOfSkipLevels, docCount)
+ {
+ this.freqOutput = freqOutput;
+ this.proxOutput = proxOutput;
+
+ lastSkipDoc = new int[numberOfSkipLevels];
+ lastSkipPayloadLength = new int[numberOfSkipLevels];
+ lastSkipFreqPointer = new long[numberOfSkipLevels];
+ lastSkipProxPointer = new long[numberOfSkipLevels];
+ }
+
+ /// <summary> Sets the values for the current skip data. </summary>
+ internal virtual void SetSkipData(int doc, bool storePayloads, int payloadLength)
+ {
+ this.curDoc = doc;
+ this.curStorePayloads = storePayloads;
+ this.curPayloadLength = payloadLength;
+ this.curFreqPointer = freqOutput.GetFilePointer();
+ this.curProxPointer = proxOutput.GetFilePointer();
+ }
+
+ protected internal override void ResetSkip()
+ {
+ base.ResetSkip();
+ for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0;
+ for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list
+ for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.GetFilePointer();
+ for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.GetFilePointer();
+ }
+
+ protected internal override void WriteSkipData(int level, IndexOutput skipBuffer)
+ {
+ // To efficiently store payloads in the posting lists we do not store the length of
+ // every payload. Instead we omit the length for a payload if the previous payload had
+ // the same length.
+ // However, in order to support skipping the payload length at every skip point must be known.
+ // So we use the same length encoding that we use for the posting lists for the skip data as well:
+ // Case 1: current field does not store payloads
+ // SkipDatum --> DocSkip, FreqSkip, ProxSkip
+ // DocSkip,FreqSkip,ProxSkip --> VInt
+ // DocSkip records the document number before every SkipInterval th document in TermFreqs.
+ // Document numbers are represented as differences from the previous value in the sequence.
+ // Case 2: current field stores payloads
+ // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip
+ // DocSkip,FreqSkip,ProxSkip --> VInt
+ // PayloadLength --> VInt
+ // In this case DocSkip/2 is the difference between
+ // the current and the previous value. If DocSkip
+ // is odd, then a PayloadLength encoded as VInt follows,
+ // if DocSkip is even, then it is assumed that the
+ // current payload length equals the length at the previous
+ // skip point
+ if (curStorePayloads)
+ {
+ int delta = curDoc - lastSkipDoc[level];
+ if (curPayloadLength == lastSkipPayloadLength[level])
+ {
+ // the current payload length equals the length at the previous skip point,
+ // so we don't store the length again
+ skipBuffer.WriteVInt(delta * 2);
+ }
+ else
+ {
+ // the payload length is different from the previous one. We shift the DocSkip,
+ // set the lowest bit and store the current payload length as VInt.
+ skipBuffer.WriteVInt(delta * 2 + 1);
+ skipBuffer.WriteVInt(curPayloadLength);
+ lastSkipPayloadLength[level] = curPayloadLength;
+ }
+ }
+ else
+ {
+ // current field does not store payloads
+ skipBuffer.WriteVInt(curDoc - lastSkipDoc[level]);
+ }
+ skipBuffer.WriteVInt((int) (curFreqPointer - lastSkipFreqPointer[level]));
+ skipBuffer.WriteVInt((int) (curProxPointer - lastSkipProxPointer[level]));
+
+ lastSkipDoc[level] = curDoc;
+ //System.out.println("write doc at level " + level + ": " + curDoc);
+
+ lastSkipFreqPointer[level] = curFreqPointer;
+ lastSkipProxPointer[level] = curProxPointer;
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DirectoryIndexReader.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Directory = Lucene.Net.Store.Directory;
+using Lock = Lucene.Net.Store.Lock;
+using LockObtainFailedException = Lucene.Net.Store.LockObtainFailedException;
+
+namespace Lucene.Net.Index
+{
+
+ /// <summary> IndexReader implementation that has access to a Directory.
+ /// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
+ /// "own" the directory, which means that they try to acquire a write lock
+ /// whenever index modifications are performed.
+ /// </summary>
+ abstract public class DirectoryIndexReader : IndexReader
+ {
+ private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
+ {
+ private void InitBlock(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy)
+ {
+ this.closeDirectory = closeDirectory;
+ this.deletionPolicy = deletionPolicy;
+ }
+ private bool closeDirectory;
+ private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
+ internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1) : base(Param1)
+ {
+ InitBlock(closeDirectory, deletionPolicy);
+ }
+
+ protected internal override System.Object DoBody(System.String segmentFileName)
+ {
+
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(directory, segmentFileName);
+
+ DirectoryIndexReader reader;
+
+ if (infos.Count == 1)
+ {
+ // index is optimized
+ reader = SegmentReader.Get(infos, infos.Info(0), closeDirectory);
+ }
+ else
+ {
+ reader = new MultiSegmentReader(directory, infos, closeDirectory);
+ }
+ reader.SetDeletionPolicy(deletionPolicy);
+ return reader;
+ }
+ }
+
+ private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
+ {
+ private void InitBlock(DirectoryIndexReader enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private DirectoryIndexReader enclosingInstance;
+ public DirectoryIndexReader Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1) : base(Param1)
+ {
+ InitBlock(enclosingInstance);
+ }
+
+ protected internal override System.Object DoBody(System.String segmentFileName)
+ {
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(directory, segmentFileName);
+
+ DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
+
+ if (Enclosing_Instance != newReader)
+ {
+ newReader.Init(directory, infos, Enclosing_Instance.closeDirectory);
+ newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
+ }
+
+ return newReader;
+ }
+ }
+ protected internal Directory directory;
+ protected internal bool closeDirectory;
+ private IndexDeletionPolicy deletionPolicy;
+
+ private SegmentInfos segmentInfos;
+ private Lock writeLock;
+ private bool stale;
+
+ /// <summary>Used by commit() to record pre-commit state in case
+ /// rollback is necessary
+ /// </summary>
+ private bool rollbackHasChanges;
+ private SegmentInfos rollbackSegmentInfos;
+
+
+ internal virtual void Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory)
+ {
+ this.directory = directory;
+ this.segmentInfos = segmentInfos;
+ this.closeDirectory = closeDirectory;
+ }
+
+ protected internal DirectoryIndexReader()
+ {
+ }
+
+ internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory) : base()
+ {
+ Init(directory, segmentInfos, closeDirectory);
+ }
+
+ internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
+ {
+
+ return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory).Run();
+ }
+
+
+ public override IndexReader Reopen()
+ {
+ lock (this)
+ {
+ EnsureOpen();
+
+ if (this.hasChanges || this.IsCurrent())
+ {
+ // the index hasn't changed - nothing to do here
+ return this;
+ }
+
+ return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile1(this, directory).Run();
+ }
+ }
+
+ /// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
+ protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
+
+ public virtual void SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
+ {
+ this.deletionPolicy = deletionPolicy;
+ }
+
+ /// <summary>Returns the directory this index resides in.</summary>
+ public override Directory Directory()
+ {
+ EnsureOpen();
+ return directory;
+ }
+
+ /// <summary> Version number when this IndexReader was opened.</summary>
+ public override long GetVersion()
+ {
+ EnsureOpen();
+ return segmentInfos.GetVersion();
+ }
+
+ /// <summary> Check whether this IndexReader is still using the
+ /// current (i.e., most recently committed) version of the
+ /// index. If a writer has committed any changes to the
+ /// index since this reader was opened, this will return
+ /// <code>false</code>, in which case you must open a new
+ /// IndexReader in order to see the changes. See the
+ /// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
+ /// flag which controls when the {@link IndexWriter}
+ /// actually commits changes to the index.
+ ///
+ /// </summary>
+ /// <throws> CorruptIndexException if the index is corrupt </throws>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ public override bool IsCurrent()
+ {
+ EnsureOpen();
+ return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
+ }
+
+ /// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
+ /// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
+ /// </returns>
+ public override bool IsOptimized()
+ {
+ EnsureOpen();
+ return segmentInfos.Count == 1 && HasDeletions() == false;
+ }
+
+ protected internal override void DoClose()
+ {
+ if (closeDirectory)
+ directory.Close();
+ }
+
+ /// <summary> Commit changes resulting from delete, undeleteAll, or
+ /// setNorm operations
+ ///
+ /// If an exception is hit, then either no changes or all
+ /// changes will have been committed to the index
+ /// (transactional semantics).
+ /// </summary>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ protected internal override void DoCommit()
+ {
+ if (hasChanges)
+ {
+ if (segmentInfos != null)
+ {
+
+ // Default deleter (for backwards compatibility) is
+ // KeepOnlyLastCommitDeleter:
+ IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
+
+ // Checkpoint the state we are about to change, in
+ // case we have to roll back:
+ StartCommit();
+
+ bool success = false;
+ try
+ {
+ CommitChanges();
+ segmentInfos.Write(directory);
+ success = true;
+ }
+ finally
+ {
+
+ if (!success)
+ {
+
+ // Rollback changes that were made to
+ // SegmentInfos but failed to get [fully]
+ // committed. This way this reader instance
+ // remains consistent (matched to what's
+ // actually in the index):
+ RollbackCommit();
+
+ // Recompute deletable files & remove them (so
+ // partially written .del files, etc, are
+ // removed):
+ deleter.Refresh();
+ }
+ }
+
+ // Have the deleter remove any now unreferenced
+ // files due to this commit:
+ deleter.Checkpoint(segmentInfos, true);
+
+ if (writeLock != null)
+ {
+ writeLock.Release(); // release write lock
+ writeLock = null;
+ }
+ }
+ else
+ CommitChanges();
+ }
+ hasChanges = false;
+ }
+
+ protected internal abstract void CommitChanges();
+
+ /// <summary> Tries to acquire the WriteLock on this directory.
+ /// this method is only valid if this IndexReader is directory owner.
+ ///
+ /// </summary>
+ /// <throws> StaleReaderException if the index has changed </throws>
+ /// <summary> since this reader was opened
+ /// </summary>
+ /// <throws> CorruptIndexException if the index is corrupt </throws>
+ /// <throws> LockObtainFailedException if another writer </throws>
+ /// <summary> has this index open (<code>write.lock</code> could not
+ /// be obtained)
+ /// </summary>
+ /// <throws> IOException if there is a low-level IO error </throws>
+ protected internal override void AcquireWriteLock()
+ {
+ if (segmentInfos != null)
+ {
+ EnsureOpen();
+ if (stale)
+ throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+
+ if (this.writeLock == null)
+ {
+ Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
+ if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
+ // obtain write lock
+ {
+ throw new LockObtainFailedException("Index locked for write: " + writeLock);
+ }
+ this.writeLock = writeLock;
+
+ // we have to check whether index has changed since this reader was opened.
+ // if so, this reader is no longer valid for deletion
+ if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
+ {
+ stale = true;
+ this.writeLock.Release();
+ this.writeLock = null;
+ throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+ }
+ }
+ }
+ }
+
+ /// <summary> Should internally checkpoint state that will change
+ /// during commit so that we can rollback if necessary.
+ /// </summary>
+ internal virtual void StartCommit()
+ {
+ if (segmentInfos != null)
+ {
+ rollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
+ }
+ rollbackHasChanges = hasChanges;
+ }
+
+ /// <summary> Rolls back state to just before the commit (this is
+ /// called by commit() if there is some exception while
+ /// committing).
+ /// </summary>
+ internal virtual void RollbackCommit()
+ {
+ if (segmentInfos != null)
+ {
+ for (int i = 0; i < segmentInfos.Count; i++)
+ {
+ // Rollback each segmentInfo. Because the
+ // SegmentReader holds a reference to the
+ // SegmentInfo we can't [easily] just replace
+ // segmentInfos, so we reset it in place instead:
+ segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
+ }
+ rollbackSegmentInfos = null;
+ }
+
+ hasChanges = rollbackHasChanges;
+ }
+
+ /// <summary>Release the write lock, if needed. </summary>
+ ~DirectoryIndexReader()
+ {
+ try
+ {
+ if (writeLock != null)
+ {
+ writeLock.Release(); // release write lock
+ writeLock = null;
+ }
+ }
+ finally
+ {
+ // {{Aroush-2.3.1}} do we need to call Finalize() here?
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocumentWriter.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs Tue Jun 24 19:52:22 2008
@@ -14,7 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+// {{Aroush-2.3.1}} remove this file from SVN
+/*
using System;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using Token = Lucene.Net.Analysis.Token;
@@ -509,4 +510,4 @@
offsets = null;
}
}
-}
\ No newline at end of file
+}*/
\ No newline at end of file