You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@lucenenet.apache.org by Mark Burks <bi...@gmail.com> on 2006/12/20 16:56:06 UTC
last n documents not written to index
When I am creating an index the last 8 documents are not being added.
I am using IndexWriter (CreateCacheWriter) to write the index to a RAM
directory. And then I use a IndexWriter to write the index to file.
When I call IndexWriter.AddIndexes(new Directory[] {cache} ) the cache
contains 1418 documents. When IndexWriter is finished optimizing I only have
1410 documents in my index.
If I vary the MergeFactor there are still documents missing. For example
if I change the MergeFactor to 2 and the MaxBufferedDocs to 2 and process
only 20 documents I am missing the last 2 documents. I created an index for
20 documents with the defaults for MergeFactor (10) and MaxBufferedDocs 10
and the last 10 documents were missing.
Any ideas?
Using Lucene.Net version 1.9.1 build 6
This is the code I am using to do this.
public class GenericIndexer
{
protected static readonly log4net.ILog log =
log4net.LogManager.GetLogger(System.Reflection.MethodBase
.GetCurrentMethod().DeclaringType);
const int ProgressInterval = 5000;
const int RamIndexFlushInterval = 150000; //300000;
const int GCCollectInterval = 1000;
private Directory destDirectory;
private Analyzer analyzer;
public GenericIndexer(Directory destDirectory, Analyzer
analyzer)
{
this.destDirectory = destDirectory;
this.analyzer = analyzer;
}
public void PerformIndexing(IDocumentSource docSource)
{
PerformIndexing(docSource, Int32.MaxValue);
}
public void PerformIndexing(IDocumentSource docSource,
intmaxRowsToIndex)
{
//IndexWriter fsWriter = new IndexWriter(fd, analyzer,
true);
// set up the RAM directory and writer
// this will cache writes in memory which will be flushed
to disk
// every RamIndexFlushInterval rows
Directory rd = CreateCacheDirectory();
IndexWriter ramWriter = CreateCacheWriter(rd);
IndexWriter destWriter = new IndexWriter(destDirectory,
analyzer, true);
TimeSpan blockTime;
TimeSpan overallTime;
double overallDocsPerSec;
double blockDocsPerSec;
Document thisDoc;
double docCount = 0;
DateTime startTime = DateTime.Now;
DateTime blockStartTime = startTime;
// keep going while we have documents
while (((thisDoc = docSource.GetNextDocument()) != null)
&& docCount < maxRowsToIndex)
{
ramWriter.AddDocument(thisDoc);
docCount++;
// are we ready to flush the RAM cache?
if ((docCount % RamIndexFlushInterval) == 0)
{
log.Info("Preparing to flush the indexing RAM
cache");
DateTime start = DateTime.Now;
FlushCache(rd, destWriter);
TimeSpan time = DateTime.Now - start;
log.Info("Took " +
time.TotalMilliseconds.ToString() + " milliseconds to flush.");
// create a new RAMDirectory and ramWriter
rd.Close();
rd = CreateCacheDirectory();
ramWriter = CreateCacheWriter(rd);
log.Info("indexing RAM cache");
}
// Should we report progress?
if ((docCount % ProgressInterval) == 0)
{
blockTime = DateTime.Now - blockStartTime;
overallTime = DateTime.Now - startTime;
overallDocsPerSec = (docCount * 1000) /
overallTime.TotalMilliseconds;
blockDocsPerSec = (ProgressInterval * 1000) /
blockTime.TotalMilliseconds;
log.Info(string.Format("{0} documents indexed;
docs/sec: {1:0.0} overall; {2:0.0} current",
docCount, overallDocsPerSec,
blockDocsPerSec));
blockStartTime = DateTime.Now;
}
}
// flush out any remaining items in the cache
if (ramWriter.DocCount() > 0)
{
FlushCache(rd, destWriter);
rd.Close();
}
destWriter.Close();
}
/// <summary>
/// Flush items out of the RAM directory cache by writing them
to the file system directory.
/// </summary>
/// <param name="rd"></param>
/// <param name="fd"></param>
//private void FlushRamCache(RAMDirectory rd, FSDirectory fd)
//{
//}
/// <summary>
/// Create a new cache Directory object
/// </summary>
/// <returns></returns>
private Directory CreateCacheDirectory()
{
return new RAMDirectory();
}
/// <summary>
/// Build a new caching IndexWriter
/// </summary>
/// <param name="d"></param>
/// <returns></returns>
private IndexWriter CreateCacheWriter(Directory d)
{
IndexWriter iw = new IndexWriter(d, analyzer, true);
iw.mergeFactor = 1000;
return iw;
}
/// <summary>
/// Flushes the RAM cache to the destination directory
/// </summary>
/// <param name="cache"></param>
/// <param name="destWriter"></param>
private void FlushCache(Directory cache, IndexWriter destWriter)
{
destWriter.AddIndexes(new Directory[] { cache });
}
}
Mark
Re: last n documents not written to index
Posted by Mark Burks <bi...@gmail.com>.
I found the problem. Before calling FlushCache I needed ramWriter.Close();
in order for newly added documents to get flushed before the index merge.
Thanks for reading,
Mark
On 12/20/06, Mark Burks <bi...@gmail.com> wrote:
>
> When I am creating an index the last 8 documents are not being added.
>
> I am using IndexWriter (CreateCacheWriter) to write the index to a RAM
> directory. And then I use a IndexWriter to write the index to file.
>
> When I call IndexWriter.AddIndexes(new Directory[] {cache} ) the cache
> contains 1418 documents. When IndexWriter is finished optimizing I only have
> 1410 documents in my index.
>
> If I vary the MergeFactor there are still documents missing. For example
> if I change the MergeFactor to 2 and the MaxBufferedDocs to 2 and process
> only 20 documents I am missing the last 2 documents. I created an index for
> 20 documents with the defaults for MergeFactor (10) and MaxBufferedDocs 10
> and the last 10 documents were missing.
>
>
> Any ideas?
>
> Using Lucene.Net version 1.9.1 build 6
>
>
>
> This is the code I am using to do this.
>
>
>
> public class GenericIndexer
>
> {
>
> protected static readonly log4net.ILog log =
> log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
>
>
>
>
> const int ProgressInterval = 5000;
>
> const int RamIndexFlushInterval = 150000; //300000;
>
> const int GCCollectInterval = 1000;
>
>
>
> private Directory destDirectory;
>
> private Analyzer analyzer;
>
>
>
> public GenericIndexer(Directory destDirectory, Analyzer
> analyzer)
>
> {
>
> this.destDirectory = destDirectory;
>
> this.analyzer = analyzer;
>
> }
>
>
>
> public void PerformIndexing(IDocumentSource docSource)
>
> {
>
> PerformIndexing(docSource, Int32.MaxValue);
>
> }
>
>
>
> public void PerformIndexing(IDocumentSource docSource, intmaxRowsToIndex)
>
> {
>
> //IndexWriter fsWriter = new IndexWriter(fd, analyzer,
> true);
>
>
>
> // set up the RAM directory and writer
>
> // this will cache writes in memory which will be
> flushed to disk
>
> // every RamIndexFlushInterval rows
>
> Directory rd = CreateCacheDirectory();
>
> IndexWriter ramWriter = CreateCacheWriter(rd);
>
> IndexWriter destWriter = new IndexWriter(destDirectory,
> analyzer, true);
>
>
>
> TimeSpan blockTime;
>
> TimeSpan overallTime;
>
> double overallDocsPerSec;
>
> double blockDocsPerSec;
>
>
>
> Document thisDoc;
>
> double docCount = 0;
>
> DateTime startTime = DateTime.Now;
>
> DateTime blockStartTime = startTime;
>
> // keep going while we have documents
>
> while (((thisDoc = docSource.GetNextDocument()) != null)
> && docCount < maxRowsToIndex)
>
> {
>
> ramWriter.AddDocument (thisDoc);
>
> docCount++;
>
>
>
> // are we ready to flush the RAM cache?
>
> if ((docCount % RamIndexFlushInterval) == 0)
>
> {
>
> log.Info("Preparing to flush the indexing
> RAM cache");
>
>
>
> DateTime start = DateTime.Now;
>
> FlushCache(rd, destWriter);
>
> TimeSpan time = DateTime.Now - start;
>
>
>
> log.Info("Took " +
> time.TotalMilliseconds.ToString() + " milliseconds to flush.");
>
>
>
> // create a new RAMDirectory and ramWriter
>
> rd.Close();
>
> rd = CreateCacheDirectory();
>
> ramWriter = CreateCacheWriter(rd);
>
>
>
> log.Info("indexing RAM cache");
>
> }
>
>
>
> // Should we report progress?
>
> if ((docCount % ProgressInterval) == 0)
>
> {
>
> blockTime = DateTime.Now - blockStartTime;
>
> overallTime = DateTime.Now - startTime;
>
> overallDocsPerSec = (docCount * 1000) /
> overallTime.TotalMilliseconds;
>
> blockDocsPerSec = (ProgressInterval * 1000)
> / blockTime.TotalMilliseconds;
> log.Info(string.Format("{0} documents
> indexed; docs/sec: {1:0.0} overall; {2:0.0} current",
> docCount, overallDocsPerSec,
> blockDocsPerSec));
>
> blockStartTime = DateTime.Now;
>
> }
>
>
>
> }
>
>
>
> // flush out any remaining items in the cache
>
> if (ramWriter.DocCount() > 0)
>
> {
>
> FlushCache(rd, destWriter);
>
> rd.Close ();
>
> }
>
>
>
> destWriter.Close ();
>
> }
>
>
>
> /// <summary>
>
> /// Flush items out of the RAM directory cache by writing them
> to the file system directory.
>
> /// </summary>
>
> /// <param name="rd"></param>
>
> /// <param name="fd"></param>
>
> //private void FlushRamCache(RAMDirectory rd, FSDirectory fd)
>
> //{
>
>
>
> //}
>
>
>
> /// <summary>
>
> /// Create a new cache Directory object
>
> /// </summary>
>
> /// <returns></returns>
>
> private Directory CreateCacheDirectory()
>
> {
>
> return new RAMDirectory();
>
> }
>
>
>
> /// <summary>
>
> /// Build a new caching IndexWriter
>
> /// </summary>
>
> /// <param name="d"></param>
>
> /// <returns></returns>
>
> private IndexWriter CreateCacheWriter(Directory d)
>
> {
>
> IndexWriter iw = new IndexWriter(d, analyzer, true);
>
> iw.mergeFactor = 1000;
>
> return iw;
>
> }
>
>
>
> /// <summary>
>
> /// Flushes the RAM cache to the destination directory
>
> /// </summary>
>
> /// <param name="cache"></param>
>
> /// <param name="destWriter"></param>
>
> private void FlushCache(Directory cache, IndexWriter
> destWriter)
>
> {
>
> destWriter.AddIndexes (new Directory[] { cache });
>
> }
>
>
>
>
>
> }
>
> Mark
>
>
>