You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@lucenenet.apache.org by Tim Haughton <ti...@gmail.com> on 2008/07/12 11:53:17 UTC

Failing to index

Noob question, I think I'm fundamentally failing to grok something here.
Trying to write a small document management tool, but the indexing isn't
playing ball. I'm reading the Manning Lucene book, and I'm not sure if
something's being lost in translation. In the app, the class that represents
a document has 3 properties that we're interesting in, all strings:

Content (full text content)
DocumentID (Unique ID string)
Folder (the ID of the folder it sits in)

But I can't get it to work. Here's an example app showing my problem:

static void Main(string[] args)
        {
            DoStuff();
        }

        private static void DoStuff()
        {
            Random r = new Random();

            if (!Directory.Exists("C:\\MyLuceneTest"))
            {
                Console.WriteLine("Creating Lucene index directory.");
            }

            FileInfo indexFolder = new FileInfo("C:\\MyLuceneTest");
            IndexWriter writer = new IndexWriter(indexFolder, new
StandardAnalyzer(), true);
            writer.SetUseCompoundFile(true);

            Document doc = new Document();

            doc.Add(new Field("content", r.Next().ToString(), Field.Store.NO,
Field.Index.TOKENIZED));
            doc.Add(new Field("documentID", r.Next().ToString() ,
Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
            doc.Add(new Field("folder", "Inbox",
Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

            Document doc2 = new Document();
            doc2.Add(new Field("content", r.Next().ToString(),
Field.Store.YES, Field.Index.TOKENIZED));
            doc2.Add(new Field("documentID", r.Next().ToString(),
Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("folder", "Inbox",
Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

            writer.AddDocument(doc);
            writer.AddDocument(doc2);
            writer.Optimize();
            writer.Close();

            Console.WriteLine("Added document to index...let's try and find
it.");

            IndexSearcher searcher = new IndexSearcher("C:\\MyLuceneTest");
            Term t = new Term("folder", "Inbox");

            Console.WriteLine("Searching for a document with the 'folder'
field set to 'Inbox'");

            Query query = new TermQuery(t);
            Hits hits = searcher.Search(query);
            searcher.Close();

            Console.WriteLine("Number of hits: " +
hits.Length().ToString());
        }

This produces zero hits.

If I change the folder field to UNTOKENIZED, I get 2 hits. But if I run the
app again, I still get 2 hits. Shouldn't I be getting 4, then 6?
Fundamentally missing part of the puzzle.


Tim

Re: Failing to index

Posted by Tim Haughton <ti...@gmail.com>.
Hi Digy, thanks for the response.

I didn't realise that about the StandardAnalyzer. The new documents
seemingly disappearing each time was of course a 'true' being passed to the
create parameter on the index writer.

Thanks again,

Tim

2008/7/12 Digy <di...@gmail.com>:

> Since you are indexing with StandardAnalyzer, all tokens of indexed fields
> are stored in lowercase.
>
> if you create the term like
>
>      Term t = new Term("folder", "inbox");
>
> you will get results.
>
>
>
> Or alternatively, you can use QueryParser
>
>
>
> DIGY
>
>
>
>
>
> -----Original Message-----
> From: Tim Haughton [mailto:timhaughton@gmail.com]
> Sent: Saturday, July 12, 2008 12:53 PM
> To: lucene-net-user@incubator.apache.org
> Subject: Failing to index
>
>
>
> Noob question, I think I'm fundamentally failing to grok something here.
>
> Trying to write a small document management tool, but the indexing isn't
>
> playing ball. I'm reading the Manning Lucene book, and I'm not sure if
>
> something's being lost in translation. In the app, the class that
> represents
>
> a document has 3 properties that we're interesting in, all strings:
>
>
>
> Content (full text content)
>
> DocumentID (Unique ID string)
>
> Folder (the ID of the folder it sits in)
>
>
>
> But I can't get it to work. Here's an example app showing my problem:
>
>
>
> static void Main(string[] args)
>
>        {
>
>            DoStuff();
>
>        }
>
>
>
>        private static void DoStuff()
>
>        {
>
>            Random r = new Random();
>
>
>
>            if (!Directory.Exists("C:\\MyLuceneTest"))
>
>            {
>
>                Console.WriteLine("Creating Lucene index directory.");
>
>            }
>
>
>
>            FileInfo indexFolder = new FileInfo("C:\\MyLuceneTest");
>
>            IndexWriter writer = new IndexWriter(indexFolder, new
>
> StandardAnalyzer(), true);
>
>            writer.SetUseCompoundFile(true);
>
>
>
>            Document doc = new Document();
>
>
>
>            doc.Add(new Field("content", r.Next().ToString(),
> Field.Store.NO,
>
> Field.Index.TOKENIZED));
>
>            doc.Add(new Field("documentID", r.Next().ToString() ,
>
> Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
>
>            doc.Add(new Field("folder", "Inbox",
>
> Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));
>
>
>
>            Document doc2 = new Document();
>
>            doc2.Add(new Field("content", r.Next().ToString(),
>
> Field.Store.YES, Field.Index.TOKENIZED));
>
>            doc2.Add(new Field("documentID", r.Next().ToString(),
>
> Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
>
>            doc2.Add(new Field("folder", "Inbox",
>
> Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));
>
>
>
>            writer.AddDocument(doc);
>
>            writer.AddDocument(doc2);
>
>            writer.Optimize();
>
>            writer.Close();
>
>
>
>            Console.WriteLine("Added document to index...let's try and find
>
> it.");
>
>
>
>            IndexSearcher searcher = new IndexSearcher("C:\\MyLuceneTest");
>
>            Term t = new Term("folder", "Inbox");
>
>
>
>            Console.WriteLine("Searching for a document with the 'folder'
>
> field set to 'Inbox'");
>
>
>
>            Query query = new TermQuery(t);
>
>            Hits hits = searcher.Search(query);
>
>            searcher.Close();
>
>
>
>            Console.WriteLine("Number of hits: " +
>
> hits.Length().ToString());
>
>        }
>
>
>
> This produces zero hits.
>
>
>
> If I change the folder field to UNTOKENIZED, I get 2 hits. But if I run the
>
> app again, I still get 2 hits. Shouldn't I be getting 4, then 6?
>
> Fundamentally missing part of the puzzle.
>
>
>
>
>
> Tim
>
>

RE: Failing to index

Posted by Digy <di...@gmail.com>.
Since you are indexing with StandardAnalyzer, all tokens of indexed fields
are stored in lowercase.

if you create the term like            

      Term t = new Term("folder", "inbox"); 

you will get results.

 

Or alternatively, you can use QueryParser

 

DIGY

 

 

-----Original Message-----
From: Tim Haughton [mailto:timhaughton@gmail.com] 
Sent: Saturday, July 12, 2008 12:53 PM
To: lucene-net-user@incubator.apache.org
Subject: Failing to index

 

Noob question, I think I'm fundamentally failing to grok something here.

Trying to write a small document management tool, but the indexing isn't

playing ball. I'm reading the Manning Lucene book, and I'm not sure if

something's being lost in translation. In the app, the class that represents

a document has 3 properties that we're interesting in, all strings:

 

Content (full text content)

DocumentID (Unique ID string)

Folder (the ID of the folder it sits in)

 

But I can't get it to work. Here's an example app showing my problem:

 

static void Main(string[] args)

        {

            DoStuff();

        }

 

        private static void DoStuff()

        {

            Random r = new Random();

 

            if (!Directory.Exists("C:\\MyLuceneTest"))

            {

                Console.WriteLine("Creating Lucene index directory.");

            }

 

            FileInfo indexFolder = new FileInfo("C:\\MyLuceneTest");

            IndexWriter writer = new IndexWriter(indexFolder, new

StandardAnalyzer(), true);

            writer.SetUseCompoundFile(true);

 

            Document doc = new Document();

 

            doc.Add(new Field("content", r.Next().ToString(),
Field.Store.NO,

Field.Index.TOKENIZED));

            doc.Add(new Field("documentID", r.Next().ToString() ,

Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));

            doc.Add(new Field("folder", "Inbox",

Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

 

            Document doc2 = new Document();

            doc2.Add(new Field("content", r.Next().ToString(),

Field.Store.YES, Field.Index.TOKENIZED));

            doc2.Add(new Field("documentID", r.Next().ToString(),

Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));

            doc2.Add(new Field("folder", "Inbox",

Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

 

            writer.AddDocument(doc);

            writer.AddDocument(doc2);

            writer.Optimize();

            writer.Close();

 

            Console.WriteLine("Added document to index...let's try and find

it.");

 

            IndexSearcher searcher = new IndexSearcher("C:\\MyLuceneTest");

            Term t = new Term("folder", "Inbox");

 

            Console.WriteLine("Searching for a document with the 'folder'

field set to 'Inbox'");

 

            Query query = new TermQuery(t);

            Hits hits = searcher.Search(query);

            searcher.Close();

 

            Console.WriteLine("Number of hits: " +

hits.Length().ToString());

        }

 

This produces zero hits.

 

If I change the folder field to UNTOKENIZED, I get 2 hits. But if I run the

app again, I still get 2 hits. Shouldn't I be getting 4, then 6?

Fundamentally missing part of the puzzle.

 

 

Tim


RE: Failing to index

Posted by Digy <di...@gmail.com>.
Also, since the third parameter of the IndexWriter's constructor( createNew)
is true you get 2 results everytime you run the app.

DIGY


-----Original Message-----
From: Tim Haughton [mailto:timhaughton@gmail.com] 
Sent: Saturday, July 12, 2008 12:53 PM
To: lucene-net-user@incubator.apache.org
Subject: Failing to index

Noob question, I think I'm fundamentally failing to grok something here.
Trying to write a small document management tool, but the indexing isn't
playing ball. I'm reading the Manning Lucene book, and I'm not sure if
something's being lost in translation. In the app, the class that represents
a document has 3 properties that we're interesting in, all strings:

Content (full text content)
DocumentID (Unique ID string)
Folder (the ID of the folder it sits in)

But I can't get it to work. Here's an example app showing my problem:

static void Main(string[] args)
        {
            DoStuff();
        }

        private static void DoStuff()
        {
            Random r = new Random();

            if (!Directory.Exists("C:\\MyLuceneTest"))
            {
                Console.WriteLine("Creating Lucene index directory.");
            }

            FileInfo indexFolder = new FileInfo("C:\\MyLuceneTest");
            IndexWriter writer = new IndexWriter(indexFolder, new
StandardAnalyzer(), true);
            writer.SetUseCompoundFile(true);

            Document doc = new Document();

            doc.Add(new Field("content", r.Next().ToString(),
Field.Store.NO,
Field.Index.TOKENIZED));
            doc.Add(new Field("documentID", r.Next().ToString() ,
Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
            doc.Add(new Field("folder", "Inbox",
Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

            Document doc2 = new Document();
            doc2.Add(new Field("content", r.Next().ToString(),
Field.Store.YES, Field.Index.TOKENIZED));
            doc2.Add(new Field("documentID", r.Next().ToString(),
Lucene.Net.Documents.Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("folder", "Inbox",
Lucene.Net.Documents.Field.Store.YES, Field.Index.TOKENIZED));

            writer.AddDocument(doc);
            writer.AddDocument(doc2);
            writer.Optimize();
            writer.Close();

            Console.WriteLine("Added document to index...let's try and find
it.");

            IndexSearcher searcher = new IndexSearcher("C:\\MyLuceneTest");
            Term t = new Term("folder", "Inbox");

            Console.WriteLine("Searching for a document with the 'folder'
field set to 'Inbox'");

            Query query = new TermQuery(t);
            Hits hits = searcher.Search(query);
            searcher.Close();

            Console.WriteLine("Number of hits: " +
hits.Length().ToString());
        }

This produces zero hits.

If I change the folder field to UNTOKENIZED, I get 2 hits. But if I run the
app again, I still get 2 hits. Shouldn't I be getting 4, then 6?
Fundamentally missing part of the puzzle.


Tim