You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@lucenenet.apache.org by Mark Burks <bi...@gmail.com> on 2006/12/20 16:56:06 UTC

last n documents not written to index

When I am creating an index the last 8 documents are not being added.

I am using IndexWriter (CreateCacheWriter) to write the index to a RAM
directory. And then I use a IndexWriter to write the index to file.

When I call IndexWriter.AddIndexes(new Directory[] {cache} ) the cache
contains 1418 documents. When IndexWriter is finished optimizing I only have
1410 documents in my index.

 If I vary the MergeFactor there are still documents missing.  For example
if I change the MergeFactor to 2 and the MaxBufferedDocs to 2 and process
only 20 documents I am missing the last 2 documents. I created an index for
20 documents with the defaults for MergeFactor (10) and MaxBufferedDocs 10
and the last 10 documents were missing.


Any ideas?

Using Lucene.Net version 1.9.1 build 6



This is the code I am using to do this.



public class GenericIndexer

      {

            protected static readonly log4net.ILog log =
log4net.LogManager.GetLogger(System.Reflection.MethodBase
.GetCurrentMethod().DeclaringType);



            const int ProgressInterval = 5000;

            const int RamIndexFlushInterval = 150000; //300000;

            const int GCCollectInterval = 1000;



            private Directory destDirectory;

            private Analyzer analyzer;



            public GenericIndexer(Directory destDirectory, Analyzer
analyzer)

            {

                  this.destDirectory = destDirectory;

                  this.analyzer = analyzer;

            }



            public void PerformIndexing(IDocumentSource docSource)

            {

                  PerformIndexing(docSource, Int32.MaxValue);

            }



            public void PerformIndexing(IDocumentSource docSource,
intmaxRowsToIndex)

            {

                  //IndexWriter fsWriter = new IndexWriter(fd, analyzer,
true);



                  // set up the RAM directory and writer

                  // this will cache writes in memory which will be flushed
to disk

                  // every RamIndexFlushInterval rows

                  Directory rd = CreateCacheDirectory();

                  IndexWriter ramWriter = CreateCacheWriter(rd);

                  IndexWriter destWriter = new IndexWriter(destDirectory,
analyzer, true);



                  TimeSpan blockTime;

                  TimeSpan overallTime;

                  double overallDocsPerSec;

                  double blockDocsPerSec;



                  Document thisDoc;

                  double docCount = 0;

                  DateTime startTime = DateTime.Now;

                  DateTime blockStartTime = startTime;

                  // keep going while we have documents

                  while (((thisDoc = docSource.GetNextDocument()) != null)
&& docCount < maxRowsToIndex)

                  {

                        ramWriter.AddDocument(thisDoc);

                        docCount++;



                        // are we ready to flush the RAM cache?

                        if ((docCount % RamIndexFlushInterval) == 0)

                        {

                              log.Info("Preparing to flush the indexing RAM
cache");



                              DateTime start = DateTime.Now;

                              FlushCache(rd, destWriter);

                              TimeSpan time = DateTime.Now - start;



                              log.Info("Took " +
time.TotalMilliseconds.ToString() + " milliseconds to flush.");



                              // create a new RAMDirectory and ramWriter

                              rd.Close();

                              rd = CreateCacheDirectory();

                              ramWriter = CreateCacheWriter(rd);



                              log.Info("indexing RAM cache");

                        }



                        // Should we report progress?

                        if ((docCount % ProgressInterval) == 0)

                        {

                              blockTime = DateTime.Now - blockStartTime;

                              overallTime = DateTime.Now - startTime;

                              overallDocsPerSec = (docCount * 1000) /
overallTime.TotalMilliseconds;

                              blockDocsPerSec = (ProgressInterval * 1000) /
blockTime.TotalMilliseconds;
                              log.Info(string.Format("{0} documents indexed;
docs/sec: {1:0.0} overall; {2:0.0} current",
                                           docCount, overallDocsPerSec,
blockDocsPerSec));

                              blockStartTime = DateTime.Now;

                        }



                  }



                  // flush out any remaining items in the cache

                  if (ramWriter.DocCount() > 0)

                  {

                        FlushCache(rd, destWriter);

                        rd.Close();

                  }



                  destWriter.Close();

            }



            /// <summary>

            /// Flush items out of the RAM directory cache by writing them
to the file system directory.

            /// </summary>

            /// <param name="rd"></param>

            /// <param name="fd"></param>

            //private void FlushRamCache(RAMDirectory rd, FSDirectory fd)

            //{



            //}



            /// <summary>

            /// Create a new cache Directory object

            /// </summary>

            /// <returns></returns>

            private Directory CreateCacheDirectory()

            {

                  return new RAMDirectory();

            }



            /// <summary>

            /// Build a new caching IndexWriter

            /// </summary>

            /// <param name="d"></param>

            /// <returns></returns>

            private IndexWriter CreateCacheWriter(Directory d)

            {

                  IndexWriter iw = new IndexWriter(d, analyzer, true);

                  iw.mergeFactor = 1000;

                  return iw;

            }



            /// <summary>

            /// Flushes the RAM cache to the destination directory

            /// </summary>

            /// <param name="cache"></param>

            /// <param name="destWriter"></param>

            private void FlushCache(Directory cache, IndexWriter destWriter)

            {

                  destWriter.AddIndexes(new Directory[] { cache });

            }





      }

Mark

Re: last n documents not written to index

Posted by Mark Burks <bi...@gmail.com>.
I found the problem. Before calling FlushCache I needed ramWriter.Close();
in order for newly added documents to get flushed before the index merge.

Thanks for reading,
Mark


On 12/20/06, Mark Burks <bi...@gmail.com> wrote:
>
> When I am creating an index the last 8 documents are not being added.
>
> I am using IndexWriter (CreateCacheWriter) to write the index to a RAM
> directory. And then I use a IndexWriter to write the index to file.
>
> When I call IndexWriter.AddIndexes(new Directory[] {cache} ) the cache
> contains 1418 documents. When IndexWriter is finished optimizing I only have
> 1410 documents in my index.
>
>  If I vary the MergeFactor there are still documents missing.  For example
> if I change the MergeFactor to 2 and the MaxBufferedDocs to 2 and process
> only 20 documents I am missing the last 2 documents. I created an index for
> 20 documents with the defaults for MergeFactor (10) and MaxBufferedDocs 10
> and the last 10 documents were missing.
>
>
> Any ideas?
>
> Using Lucene.Net version 1.9.1 build 6
>
>
>
> This is the code I am using to do this.
>
>
>
> public class GenericIndexer
>
>       {
>
>             protected static readonly log4net.ILog log =
> log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
>
>
>
>
>             const int ProgressInterval = 5000;
>
>             const int RamIndexFlushInterval = 150000; //300000;
>
>             const int GCCollectInterval = 1000;
>
>
>
>             private Directory destDirectory;
>
>             private Analyzer analyzer;
>
>
>
>             public GenericIndexer(Directory destDirectory, Analyzer
> analyzer)
>
>             {
>
>                   this.destDirectory = destDirectory;
>
>                   this.analyzer = analyzer;
>
>             }
>
>
>
>             public void PerformIndexing(IDocumentSource docSource)
>
>             {
>
>                   PerformIndexing(docSource, Int32.MaxValue);
>
>             }
>
>
>
>             public void PerformIndexing(IDocumentSource docSource, intmaxRowsToIndex)
>
>             {
>
>                   //IndexWriter fsWriter = new IndexWriter(fd, analyzer,
> true);
>
>
>
>                   // set up the RAM directory and writer
>
>                   // this will cache writes in memory which will be
> flushed to disk
>
>                   // every RamIndexFlushInterval rows
>
>                   Directory rd = CreateCacheDirectory();
>
>                   IndexWriter ramWriter = CreateCacheWriter(rd);
>
>                   IndexWriter destWriter = new IndexWriter(destDirectory,
> analyzer, true);
>
>
>
>                   TimeSpan blockTime;
>
>                   TimeSpan overallTime;
>
>                   double overallDocsPerSec;
>
>                   double blockDocsPerSec;
>
>
>
>                   Document thisDoc;
>
>                   double docCount = 0;
>
>                   DateTime startTime = DateTime.Now;
>
>                   DateTime blockStartTime = startTime;
>
>                   // keep going while we have documents
>
>                   while (((thisDoc = docSource.GetNextDocument()) != null)
> && docCount < maxRowsToIndex)
>
>                   {
>
>                         ramWriter.AddDocument (thisDoc);
>
>                         docCount++;
>
>
>
>                         // are we ready to flush the RAM cache?
>
>                         if ((docCount % RamIndexFlushInterval) == 0)
>
>                         {
>
>                               log.Info("Preparing to flush the indexing
> RAM cache");
>
>
>
>                               DateTime start = DateTime.Now;
>
>                               FlushCache(rd, destWriter);
>
>                               TimeSpan time = DateTime.Now - start;
>
>
>
>                               log.Info("Took " +
> time.TotalMilliseconds.ToString() + " milliseconds to flush.");
>
>
>
>                               // create a new RAMDirectory and ramWriter
>
>                               rd.Close();
>
>                               rd = CreateCacheDirectory();
>
>                               ramWriter = CreateCacheWriter(rd);
>
>
>
>                               log.Info("indexing RAM cache");
>
>                         }
>
>
>
>                         // Should we report progress?
>
>                         if ((docCount % ProgressInterval) == 0)
>
>                         {
>
>                               blockTime = DateTime.Now - blockStartTime;
>
>                               overallTime = DateTime.Now - startTime;
>
>                               overallDocsPerSec = (docCount * 1000) /
> overallTime.TotalMilliseconds;
>
>                               blockDocsPerSec = (ProgressInterval * 1000)
> / blockTime.TotalMilliseconds;
>                               log.Info(string.Format("{0} documents
> indexed; docs/sec: {1:0.0} overall; {2:0.0} current",
>                                            docCount, overallDocsPerSec,
> blockDocsPerSec));
>
>                               blockStartTime = DateTime.Now;
>
>                         }
>
>
>
>                   }
>
>
>
>                   // flush out any remaining items in the cache
>
>                   if (ramWriter.DocCount() > 0)
>
>                   {
>
>                         FlushCache(rd, destWriter);
>
>                         rd.Close ();
>
>                   }
>
>
>
>                   destWriter.Close ();
>
>             }
>
>
>
>             /// <summary>
>
>             /// Flush items out of the RAM directory cache by writing them
> to the file system directory.
>
>             /// </summary>
>
>             /// <param name="rd"></param>
>
>             /// <param name="fd"></param>
>
>             //private void FlushRamCache(RAMDirectory rd, FSDirectory fd)
>
>             //{
>
>
>
>             //}
>
>
>
>             /// <summary>
>
>             /// Create a new cache Directory object
>
>             /// </summary>
>
>             /// <returns></returns>
>
>             private Directory CreateCacheDirectory()
>
>             {
>
>                   return new RAMDirectory();
>
>             }
>
>
>
>             /// <summary>
>
>             /// Build a new caching IndexWriter
>
>             /// </summary>
>
>             /// <param name="d"></param>
>
>             /// <returns></returns>
>
>             private IndexWriter CreateCacheWriter(Directory d)
>
>             {
>
>                   IndexWriter iw = new IndexWriter(d, analyzer, true);
>
>                   iw.mergeFactor = 1000;
>
>                   return iw;
>
>             }
>
>
>
>             /// <summary>
>
>             /// Flushes the RAM cache to the destination directory
>
>             /// </summary>
>
>             /// <param name="cache"></param>
>
>             /// <param name="destWriter"></param>
>
>             private void FlushCache(Directory cache, IndexWriter
> destWriter)
>
>             {
>
>                   destWriter.AddIndexes (new Directory[] { cache });
>
>             }
>
>
>
>
>
>       }
>
> Mark
>
>
>