You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2009/11/25 05:08:18 UTC
svn commit: r883975 [2/3] - in /lucene/pylucene/trunk: ./
java/org/apache/pylucene/search/ jcc/ samples/LuceneInAction/
samples/LuceneInAction/lia/advsearching/ samples/LuceneInAction/lia/analysis/
samples/LuceneInAction/lia/analysis/i18n/ samples/Luce...
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/queryparser/AnalysisParalysisTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/queryparser/AnalysisParalysisTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/queryparser/AnalysisParalysisTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/queryparser/AnalysisParalysisTest.py Wed Nov 25 04:08:13 2009
@@ -15,24 +15,27 @@
from lia.common.LiaTestCase import LiaTestCase
from lucene import \
- QueryParser, StandardAnalyzer, PerFieldAnalyzerWrapper, WhitespaceAnalyzer
+ QueryParser, StandardAnalyzer, PerFieldAnalyzerWrapper, \
+ WhitespaceAnalyzer, Version
class AnalysisParalysisTest(LiaTestCase):
def testAnalyzer(self):
- analyzer = StandardAnalyzer()
+ analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
queryString = "category:/philosophy/eastern"
- query = QueryParser("contents", analyzer).parse(queryString)
+ query = QueryParser(Version.LUCENE_CURRENT,
+ "contents", analyzer).parse(queryString)
self.assertEqual("category:\"philosophy eastern\"",
query.toString("contents"), "path got split, yikes!")
perFieldAnalyzer = PerFieldAnalyzerWrapper(analyzer)
perFieldAnalyzer.addAnalyzer("category", WhitespaceAnalyzer())
- query = QueryParser("contents", perFieldAnalyzer).parse(queryString)
+ query = QueryParser(Version.LUCENE_CURRENT,
+ "contents", perFieldAnalyzer).parse(queryString)
self.assertEqual("category:/philosophy/eastern",
query.toString("contents"),
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzer2.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzer2.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzer2.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzer2.py Wed Nov 25 04:08:13 2009
@@ -24,11 +24,11 @@
def __init__(self, stopWords=None):
if stopWords is None:
- self.stopWords = StopAnalyzer.ENGLISH_STOP_WORDS
+ self.stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET
else:
self.stopWords = stopWords
def tokenStream(self, fieldName, reader):
- return StopFilter(LowerCaseFilter(LetterTokenizer(reader)),
+ return StopFilter(True, LowerCaseFilter(LetterTokenizer(reader)),
self.stopWords)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerAlternativeTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerAlternativeTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerAlternativeTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerAlternativeTest.py Wed Nov 25 04:08:13 2009
@@ -24,15 +24,15 @@
def testStopAnalyzer2(self):
- tokens = AnalyzerUtils.tokensFromAnalysis(StopAnalyzer2(),
- "The quick brown...")
- AnalyzerUtils.assertTokensEqual(self, tokens, ["quick", "brown"])
+ AnalyzerUtils.assertAnalyzesTo(StopAnalyzer2(),
+ "The quick brown...",
+ ["quick", "brown"])
def testStopAnalyzerFlawed(self):
- tokens = AnalyzerUtils.tokensFromAnalysis(StopAnalyzerFlawed(),
- "The quick brown...")
- self.assertEqual("the", tokens[0].termText())
+ AnalyzerUtils.assertAnalyzesTo(StopAnalyzerFlawed(),
+ "The quick brown...",
+ ["the", "quick", "brown"])
#
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerFlawed.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerFlawed.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerFlawed.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerFlawed.py Wed Nov 25 04:08:13 2009
@@ -26,7 +26,7 @@
def __init__(self, stopWords=None):
if stopWords is None:
- self.stopWords = StopAnalyzer.ENGLISH_STOP_WORDS
+ self.stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET
else:
self.stopWords = stopWords
@@ -36,5 +36,5 @@
def tokenStream(self, fieldName, reader):
- return LowerCaseFilter(StopFilter(LetterTokenizer(reader),
+ return LowerCaseFilter(StopFilter(True, LetterTokenizer(reader),
self.stopWords))
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/stopanalyzer/StopAnalyzerTest.py Wed Nov 25 04:08:13 2009
@@ -13,7 +13,7 @@
# ====================================================================
from unittest import TestCase
-from lucene import StopAnalyzer
+from lucene import StopAnalyzer, Version
from lia.analysis.AnalyzerUtils import AnalyzerUtils
@@ -21,25 +21,17 @@
def setUp(self):
- self.stopAnalyzer = StopAnalyzer()
+ self.stopAnalyzer = StopAnalyzer(Version.LUCENE_CURRENT)
def testHoles(self):
expected = ["one", "enough"]
- AnalyzerUtils.assertTokensEqual(self,
- self.tokensFrom("one is not enough"),
- expected)
- AnalyzerUtils.assertTokensEqual(self,
- self.tokensFrom("one is enough"),
- expected)
- AnalyzerUtils.assertTokensEqual(self,
- self.tokensFrom("one enough"),
- expected)
- AnalyzerUtils.assertTokensEqual(self,
- self.tokensFrom("one but not enough"),
- expected)
-
- def tokensFrom(self, text):
-
- return AnalyzerUtils.tokensFromAnalysis(self.stopAnalyzer, text)
+ AnalyzerUtils.assertAnalyzesTo(self.stopAnalyzer, "one is not enough",
+ expected)
+ AnalyzerUtils.assertAnalyzesTo(self.stopAnalyzer, "one is enough",
+ expected)
+ AnalyzerUtils.assertAnalyzesTo(self.stopAnalyzer, "one enough",
+ expected)
+ AnalyzerUtils.assertAnalyzesTo(self.stopAnalyzer, "one but not enough",
+ expected)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzer.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzer.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzer.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzer.py Wed Nov 25 04:08:13 2009
@@ -13,7 +13,7 @@
# ====================================================================
from lucene import \
- LowerCaseFilter, StopFilter, \
+ LowerCaseFilter, StopFilter, Version, \
StandardAnalyzer, StandardTokenizer, StandardFilter, PythonAnalyzer
from lia.analysis.synonym.SynonymFilter import SynonymFilter
@@ -31,7 +31,8 @@
def tokenStream(self, fieldName, reader):
- tokenStream = LowerCaseFilter(StandardFilter(StandardTokenizer(reader)))
- tokenStream = StopFilter(tokenStream, StandardAnalyzer.STOP_WORDS)
+ tokenStream = LowerCaseFilter(StandardFilter(StandardTokenizer(Version.LUCENE_CURRENT, reader)))
+ tokenStream = StopFilter(True, tokenStream,
+ StandardAnalyzer.STOP_WORDS_SET)
return SynonymFilter(tokenStream, self.engine)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py Wed Nov 25 04:08:13 2009
@@ -16,7 +16,8 @@
from lucene import \
StandardAnalyzer, RAMDirectory, IndexWriter, Term, Document, Field, \
- IndexSearcher, TermQuery, PhraseQuery, QueryParser
+ IndexSearcher, TermQuery, PhraseQuery, QueryParser, StringReader, \
+ TermAttribute, PositionIncrementAttribute, Version
from lia.analysis.AnalyzerUtils import AnalyzerUtils
from lia.analysis.synonym.SynonymAnalyzer import SynonymAnalyzer
@@ -30,16 +31,17 @@
def setUp(self):
self.directory = RAMDirectory()
- writer = IndexWriter(self.directory, self.synonymAnalyzer, True)
+ writer = IndexWriter(self.directory, self.synonymAnalyzer, True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
doc = Document()
doc.add(Field("content",
"The quick brown fox jumps over the lazy dogs",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.close()
- self.searcher = IndexSearcher(self.directory)
+ self.searcher = IndexSearcher(self.directory, True)
def tearDown(self):
@@ -47,42 +49,54 @@
def testJumps(self):
- tokens = AnalyzerUtils.tokensFromAnalysis(self.synonymAnalyzer, "jumps")
- AnalyzerUtils.assertTokensEqual(self, tokens,
- ["jumps", "hops", "leaps"])
-
- # ensure synonyms are in the same position as the original
- self.assertEqual(1, tokens[0].getPositionIncrement(), "jumps")
- self.assertEqual(0, tokens[1].getPositionIncrement(), "hops")
- self.assertEqual(0, tokens[2].getPositionIncrement(), "leaps")
+ stream = self.synonymAnalyzer.tokenStream("contents",
+ StringReader("jumps"))
+ term = stream.addAttribute(TermAttribute.class_)
+ posIncr = stream.addAttribute(PositionIncrementAttribute.class_)
+
+ i = 0
+ expected = ["jumps", "hops", "leaps"]
+ while stream.incrementToken():
+ self.assertEqual(expected[i], term.term())
+ if i == 0:
+ expectedPos = 1
+ else:
+ expectedPos = 0
+
+ self.assertEqual(expectedPos, posIncr.getPositionIncrement())
+ i += 1
+
+ self.assertEqual(3, i)
def testSearchByAPI(self):
tq = TermQuery(Term("content", "hops"))
- hits = self.searcher.search(tq)
- self.assertEqual(1, len(hits))
+ topDocs = self.searcher.search(tq, 50)
+ self.assertEqual(1, topDocs.totalHits)
pq = PhraseQuery()
pq.add(Term("content", "fox"))
pq.add(Term("content", "hops"))
- hits = self.searcher.search(pq)
- self.assertEquals(1, len(hits))
+ topDocs = self.searcher.search(pq, 50)
+ self.assertEquals(1, topDocs.totalHits)
def testWithQueryParser(self):
- query = QueryParser("content",
+ query = QueryParser(Version.LUCENE_CURRENT, "content",
self.synonymAnalyzer).parse('"fox jumps"')
- hits = self.searcher.search(query)
+ topDocs = self.searcher.search(query, 50)
# in Lucene 1.9, position increments are no longer ignored
- self.assertEqual(1, len(hits), "!!!! what?!")
+ self.assertEqual(1, topDocs.totalHits, "!!!! what?!")
- query = QueryParser("content", StandardAnalyzer()).parse('"fox jumps"')
- hits = self.searcher.search(query)
- self.assertEqual(1, len(hits), "*whew*")
+ query = QueryParser(Version.LUCENE_CURRENT, "content",
+ StandardAnalyzer(Version.LUCENE_CURRENT)).parse('"fox jumps"')
+ topDocs = self.searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits, "*whew*")
def main(cls):
- query = QueryParser("content", cls.synonymAnalyzer).parse('"fox jumps"')
+ query = QueryParser(Version.LUCENE_CURRENT, "content",
+ cls.synonymAnalyzer).parse('"fox jumps"')
print "\"fox jumps\" parses to ", query.toString("content")
print "From AnalyzerUtils.tokensFromAnalysis: "
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymFilter.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymFilter.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymFilter.py Wed Nov 25 04:08:13 2009
@@ -12,44 +12,50 @@
# limitations under the License.
# ====================================================================
-from lucene import Token, PythonTokenFilter
+from lucene import Token, PythonTokenFilter, TermAttribute
+from lia.analysis.AnalyzerUtils import AnalyzerUtils
#
# A TokenFilter extension
#
class SynonymFilter(PythonTokenFilter):
-
TOKEN_TYPE_SYNONYM = "SYNONYM"
-
- def __init__(self, tokenStream, engine):
- super(SynonymFilter, self).__init__(tokenStream)
+ def __init__(self, inStream, engine):
+ super(SynonymFilter, self).__init__(inStream)
self.synonymStack = []
- self.input = tokenStream
+ self.termAttr = self.addAttribute(TermAttribute.class_)
+ self.save = inStream.cloneAttributes()
self.engine = engine
+ self.inStream = inStream
- def next(self):
+ def incrementToken(self):
if len(self.synonymStack) > 0:
- return self.synonymStack.pop()
+ syn = self.synonymStack.pop()
+ self.restoreState(syn)
+ return True
- # this raises StopIteration which is cleared to return null to java
- token = self.input.next()
- self.addAliasesToStack(token)
+ if not self.inStream.incrementToken():
+ return False
- return token
+ self.addAliasesToStack()
- def addAliasesToStack(self, token):
+ return True
- synonyms = self.engine.getSynonyms(token.termText())
+ def addAliasesToStack(self):
+ synonyms = self.engine.getSynonyms(self.termAttr.term())
if synonyms is None:
return
+ current = self.captureState()
+
for synonym in synonyms:
- synToken = Token(synonym, token.startOffset(), token.endOffset(),
- self.TOKEN_TYPE_SYNONYM)
- synToken.setPositionIncrement(0)
- self.synonymStack.append(synToken)
+ self.save.restoreState(current)
+ AnalyzerUtils.setTerm(self.save, synonym)
+ AnalyzerUtils.setType(self.save, self.TOKEN_TYPE_SYNONYM)
+ AnalyzerUtils.setPositionIncrement(self.save, 0)
+ self.synonymStack.append(self.save.captureState())
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py Wed Nov 25 04:08:13 2009
@@ -15,7 +15,9 @@
import os
from unittest import TestCase
-from lucene import FSDirectory, Document, System, SimpleDateFormat, Hit
+from lucene import \
+ SimpleFSDirectory, Document, \
+ System, SimpleDateFormat, File
class LiaTestCase(TestCase):
@@ -27,7 +29,7 @@
def setUp(self):
- self.directory = FSDirectory.getDirectory(self.indexDir, False)
+ self.directory = SimpleFSDirectory(File(self.indexDir))
def tearDown(self):
@@ -36,25 +38,27 @@
#
# For troubleshooting
#
- def dumpHits(self, hits):
+ def dumpHits(self, searcher, scoreDocs):
- if not hits:
+ if not scoreDocs:
print "No hits"
else:
- for hit in hits:
- hit = Hit.cast_(hit)
- print "%s: %s" %(hit.getScore(),
- hit.getDocument().get('title'))
+ for scoreDoc in scoreDocs:
+ print "%s: %s" %(scoreDoc.score,
+ searcher.doc(scoreDoc.doc).get('title'))
- def assertHitsIncludeTitle(self, hits, title):
+ def assertHitsIncludeTitle(self, searcher, scoreDocs, title,
+ fail=False):
- for hit in hits:
- doc = Hit.cast_(hit).getDocument()
+ for scoreDoc in scoreDocs:
+ doc = searcher.doc(scoreDoc.doc)
if title == doc.get("title"):
- self.assert_(True)
+ if fail:
+ self.fail("title '%s' found" %(title))
return
- self.fail("title '%s' not found" %(title))
+ if not fail:
+ self.fail("title '%s' not found" %(title))
def parseDate(self, s):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py Wed Nov 25 04:08:13 2009
@@ -15,8 +15,8 @@
import os
from lucene import \
- Document, Field, IndexWriter, StandardAnalyzer, DateField, \
- SimpleDateFormat
+ Document, Field, IndexWriter, StandardAnalyzer, NumericField, \
+ SimpleDateFormat, Version, SimpleFSDirectory, File, DateTools, DateField
# date culled from LuceneInAction.zip archive from Manning site
samplesModified = SimpleDateFormat('yyyy-MM-dd').parse('2004-12-02')
@@ -26,7 +26,10 @@
def createIndex(cls, dataDir, indexDir, useCompound):
- writer = IndexWriter(indexDir, StandardAnalyzer(), True)
+ indexDir = SimpleFSDirectory(File(indexDir))
+ writer = IndexWriter(indexDir,
+ StandardAnalyzer(Version.LUCENE_CURRENT), True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
writer.setUseCompoundFile(useCompound)
for dir, dirnames, filenames in os.walk(dataDir):
@@ -70,32 +73,45 @@
print "---------"
doc.add(Field("isbn", isbn,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("category", category,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("title", title,
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS))
+ doc.add(Field("title2", title.lower(),
+ Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS,
+ Field.TermVector.WITH_POSITIONS_OFFSETS))
# split multiple authors into unique field instances
authors = author.split(',')
for a in authors:
doc.add(Field("author", a,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS))
doc.add(Field("url", url,
- Field.Store.YES, Field.Index.NO))
+ Field.Store.YES,
+ Field.Index.NOT_ANALYZED_NO_NORMS))
doc.add(Field("subject", subject,
- Field.Store.NO, Field.Index.TOKENIZED,
- Field.TermVector.YES))
- doc.add(Field("pubmonth", pubmonth,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
- doc.add(Field("contents", ' '.join([title, subject, author]),
- Field.Store.NO, Field.Index.TOKENIZED))
+ Field.Store.NO, Field.Index.ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS))
+ doc.add(NumericField("pubmonth",
+ Field.Store.YES,
+ True).setIntValue(int(pubmonth)))
+
+ d = DateTools.stringToDate(pubmonth)
+ d = int(d.getTime() / (1000 * 3600 * 24.0))
+ doc.add(NumericField("pubmonthAsDay").setIntValue(d))
+
+ doc.add(Field("contents", ' '.join([title, subject, author, category]),
+ Field.Store.NO, Field.Index.ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS))
doc.add(Field("path", path,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("modified", DateField.dateToString(samplesModified),
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
Added: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py?rev=883975&view=auto
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py (added)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py Wed Nov 25 04:08:13 2009
@@ -0,0 +1,49 @@
+# ====================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ====================================================================
+
+from lucene import Document, IndexSearcher, PythonCollector, FieldCache
+
+#
+# A Collector extension
+#
+
+class BookLinkCollector(PythonCollector):
+
+ def __init__(self, searcher):
+ super(BookLinkCollector, self).__init__()
+
+ self.searcher = searcher
+ self.documents = {}
+
+ def acceptsDocsOutOfOrder(self):
+
+ return True
+
+ def setNextReader(self, reader, docBase):
+
+ self.docBase = docBase
+ self.urls = FieldCache.DEFAULT.getStrings(reader, "url")
+ self.titles = FieldCache.DEFAULT.getStrings(reader, "title2")
+
+ def collect(self, docID, score):
+
+ url = self.urls[docID]
+ title = self.titles[docID]
+ self.documents[url] = title
+
+ print "%s: %s" %(title, score)
+
+ def getLinks(self):
+
+ return self.documents.copy()
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py?rev=883975&view=auto
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py (added)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py Wed Nov 25 04:08:13 2009
@@ -0,0 +1,37 @@
+# ====================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ====================================================================
+
+from lia.common.LiaTestCase import LiaTestCase
+from lia.extsearch.collector.BookLinkCollector import BookLinkCollector
+
+from lucene import IndexSearcher, TermQuery, Term
+
+class CollectorTest(LiaTestCase):
+
+ def testCollecting(self):
+
+ query = TermQuery(Term("contents", "junit"))
+ searcher = IndexSearcher(self.directory, True)
+
+ collector = BookLinkCollector(searcher)
+ searcher.search(query, collector)
+
+ links = collector.getLinks()
+ self.assertEqual("java development with ant",
+ links["http://www.manning.com/antbook"])
+
+ scoreDocs = searcher.search(query, 10).scoreDocs
+ self.dumpHits(searcher, scoreDocs)
+
+ searcher.close()
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py?rev=883975&view=auto
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py (added)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py Wed Nov 25 04:08:13 2009
@@ -0,0 +1 @@
+# collector package
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/__init__.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py Wed Nov 25 04:08:13 2009
@@ -12,7 +12,8 @@
# limitations under the License.
# ====================================================================
-from lucene import IndexReader, Term, BitSet, PythonFilter, JArray
+from lucene import \
+ IndexReader, Term, BitSet, PythonFilter, JArray, OpenBitSet
#
# A Filter extension, with a TermDocs wrapper working around the lack of
@@ -25,18 +26,20 @@
super(SpecialsFilter, self).__init__()
self.accessor = accessor
- def bits(self, reader):
+ def getDocIdSet(self, reader):
- bits = BitSet(reader.maxDoc())
+ bits = OpenBitSet(long(reader.maxDoc()))
isbns = self.accessor.isbns()
+ docs = JArray(int)(1)
+ freqs = JArray(int)(1)
+
for isbn in isbns:
if isbn is not None:
termDocs = reader.termDocs(Term("isbn", isbn))
- docs = JArray(int)(1)
- freq = JArray(int)(1)
- if termDocs.read(docs, freq) == 1:
- bits.set(docs[0])
+ count = termDocs.read(docs, freqs)
+ if count == 1:
+ bits.set(long(docs[0]))
return bits
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py Wed Nov 25 04:08:13 2009
@@ -13,12 +13,12 @@
# ====================================================================
from lia.common.LiaTestCase import LiaTestCase
-from lia.extsearch.filters.MockSpecialsAccessor import MockSpecialsAccessor
+from lia.extsearch.filters.TestSpecialsAccessor import TestSpecialsAccessor
from lia.extsearch.filters.SpecialsFilter import SpecialsFilter
from lucene import \
- WildcardQuery, FilteredQuery, TermQuery, BooleanQuery, RangeQuery, \
- IndexSearcher, Term, BooleanClause
+ WildcardQuery, FilteredQuery, TermQuery, BooleanQuery, TermRangeQuery, \
+ IndexSearcher, Term, BooleanClause, MatchAllDocsQuery
class SpecialsFilterTest(LiaTestCase):
@@ -27,24 +27,23 @@
super(SpecialsFilterTest, self).setUp()
- self.allBooks = RangeQuery(Term("pubmonth", "190001"),
- Term("pubmonth", "200512"), True)
- self.searcher = IndexSearcher(self.directory)
+ self.allBooks = MatchAllDocsQuery()
+ self.searcher = IndexSearcher(self.directory, True)
def testCustomFilter(self):
isbns = ["0060812451", "0465026567"]
- accessor = MockSpecialsAccessor(isbns)
+ accessor = TestSpecialsAccessor(isbns)
filter = SpecialsFilter(accessor)
- hits = self.searcher.search(self.allBooks, filter)
- self.assertEquals(len(isbns), len(hits), "the specials")
+ topDocs = self.searcher.search(self.allBooks, filter, 50)
+ self.assertEquals(len(isbns), topDocs.totalHits, "the specials")
def testFilteredQuery(self):
isbns = ["0854402624"] # Steiner
- accessor = MockSpecialsAccessor(isbns)
+ accessor = TestSpecialsAccessor(isbns)
filter = SpecialsFilter(accessor)
educationBooks = WildcardQuery(Term("category", "*education*"))
@@ -56,6 +55,6 @@
logoOrEdBooks.add(logoBooks, BooleanClause.Occur.SHOULD)
logoOrEdBooks.add(edBooksOnSpecial, BooleanClause.Occur.SHOULD)
- hits = self.searcher.search(logoOrEdBooks)
+ topDocs = self.searcher.search(logoOrEdBooks, 50)
print logoOrEdBooks
- self.assertEqual(2, len(hits), "Papert and Steiner")
+ self.assertEqual(2, topDocs.totalHits, "Papert and Steiner")
Copied: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/TestSpecialsAccessor.py (from r820102, lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/MockSpecialsAccessor.py)
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/TestSpecialsAccessor.py?p2=lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/TestSpecialsAccessor.py&p1=lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/MockSpecialsAccessor.py&r1=820102&r2=883975&rev=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/MockSpecialsAccessor.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/TestSpecialsAccessor.py Wed Nov 25 04:08:13 2009
@@ -12,7 +12,7 @@
# limitations under the License.
# ====================================================================
-class MockSpecialsAccessor(object):
+class TestSpecialsAccessor(object):
def __init__(self, isbns):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py Wed Nov 25 04:08:13 2009
@@ -30,12 +30,13 @@
self.analyzer = WhitespaceAnalyzer()
self.directory = RAMDirectory()
- writer = IndexWriter(self.directory, self.analyzer, True)
+ writer = IndexWriter(self.directory, self.analyzer, True,
+ IndexWriter.MaxFieldLength.LIMITED)
for i in xrange(1, 501):
doc = Document()
doc.add(Field("id", NumberUtils.pad(i),
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
writer.close()
@@ -84,9 +85,9 @@
self.assertEqual("id:[0000000037 TO 0000000346]",
query.toString("field"), "padded")
- searcher = IndexSearcher(self.directory)
- hits = searcher.search(query)
- self.assertEqual(310, hits.length())
+ searcher = IndexSearcher(self.directory, True)
+ scoreDocs = searcher.search(query, 1000).scoreDocs
+ self.assertEqual(310, len(scoreDocs))
print parser.parse("special:[term TO *]")
print parser.parse("special:[* TO term]")
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/CustomQueryParser.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/CustomQueryParser.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/CustomQueryParser.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/CustomQueryParser.py Wed Nov 25 04:08:13 2009
@@ -14,8 +14,8 @@
from lucene import \
PythonQueryParser, PythonMultiFieldQueryParser, \
- PhraseQuery, RangeQuery, SpanNearQuery, SpanTermQuery, \
- Term, PhraseQuery
+ PhraseQuery, TermRangeQuery, SpanNearQuery, SpanTermQuery, \
+ Term, PhraseQuery, Version
from lia.extsearch.queryparser.NumberUtils import NumberUtils
@@ -26,7 +26,7 @@
class CustomQueryParser(PythonQueryParser):
def __init__(self, field, analyzer):
- super(CustomQueryParser, self).__init__(field, analyzer)
+ super(CustomQueryParser, self).__init__(Version.LUCENE_CURRENT, field, analyzer)
def getFuzzyQuery(self, field, termText, minSimilarity):
raise AssertionError, "Fuzzy queries not allowed"
@@ -45,24 +45,15 @@
num1 = int(part1)
num2 = int(part2)
- return RangeQuery(Term(field, NumberUtils.pad(num1)),
- Term(field, NumberUtils.pad(num2)),
- inclusive)
+ return TermRangeQuery(field,
+ NumberUtils.pad(num1),
+ NumberUtils.pad(num2),
+ inclusive, True)
if field == "special":
print part1, "->", part2
- if part1 == '*':
- t1 = None
- else:
- t1 = Term("field", part1)
-
- if part2 == '*':
- t2 = None
- else:
- t2 = Term("field", part2)
-
- return RangeQuery(t1, t2, inclusive)
+ return TermRangeQuery("field", part1, part2, inclusive, True)
return super(CustomQueryParser,
self).getRangeQuery(field, part1, part2, inclusive)
@@ -94,7 +85,7 @@
class MultiFieldCustomQueryParser(PythonMultiFieldQueryParser):
def __init__(self, fields, analyzer):
- super(MultiFieldCustomQueryParser, self).__init__(fields, analyzer)
+ super(MultiFieldCustomQueryParser, self).__init__(Version.LUCENE_CURRENT, fields, analyzer)
def getFuzzyQuery(self, super, field, termText, minSimilarity):
raise AssertionError, "Fuzzy queries not allowed"
@@ -113,24 +104,15 @@
num1 = int(part1)
num2 = int(part2)
- return RangeQuery(Term(field, NumberUtils.pad(num1)),
- Term(field, NumberUtils.pad(num2)),
- inclusive)
+ return TermRangeQuery(field,
+ NumberUtils.pad(num1),
+ NumberUtils.pad(num2),
+ inclusive, True)
if field == "special":
print part1, "->", part2
- if part1 == '*':
- t1 = None
- else:
- t1 = Term("field", part1)
-
- if part2 == '*':
- t2 = None
- else:
- t2 = Term("field", part2)
-
- return RangeQuery(t1, t2, inclusive)
+ return TermRangeQuery("field", part1, part2, inclusive, True)
return super(CustomQueryParser,
self).getRangeQuery(field, part1, part2, inclusive)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py Wed Nov 25 04:08:13 2009
@@ -13,75 +13,78 @@
# ====================================================================
from math import sqrt
-from lucene import SortField, Term, IndexReader, \
- PythonSortComparatorSource, PythonScoreDocComparator, Double
+from lucene import SortField, Term, IndexReader, FieldCache, \
+ PythonFieldComparatorSource, PythonFieldComparator, Double
#
-# A SortComparatorSource implementation
+# A FieldComparatorSource implementation
#
-class DistanceComparatorSource(PythonSortComparatorSource):
+class DistanceComparatorSource(PythonFieldComparatorSource):
def __init__(self, x, y):
super(DistanceComparatorSource, self).__init__()
+
self.x = x
self.y = y
- def newComparator(self, reader, fieldName):
+ def newComparator(self, fieldName, numHits, sortPos, reversed):
+
+ class DistanceScoreDocLookupComparator(PythonFieldComparator):
+
+ def __init__(_self, fieldName, numHits):
+ super(DistanceScoreDocLookupComparator, _self).__init__()
+ _self.values = [0.0] * numHits
+ _self.fieldName = fieldName
+
+ def setNextReader(_self, reader, docBase):
+
+ _self.xDoc = FieldCache.DEFAULT.getInts(reader, "x")
+ _self.yDoc = FieldCache.DEFAULT.getInts(reader, "y")
- #
- # A ScoreDocComparator implementation
- #
- class DistanceScoreDocLookupComparator(PythonScoreDocComparator):
-
- def __init__(self, reader, fieldName, x, y):
- super(DistanceScoreDocLookupComparator, self).__init__()
- enumerator = reader.terms(Term(fieldName, ""))
- self.distances = distances = [0.0] * reader.numDocs()
-
- if reader.numDocs() > 0:
- termDocs = reader.termDocs()
- try:
- while True:
- term = enumerator.term()
- if term is None:
- raise RuntimeError, "no terms in field %s" %(fieldName)
- if term.field() != fieldName:
- break
-
- termDocs.seek(enumerator)
- while termDocs.next():
- xy = term.text().split(',')
- deltax = int(xy[0]) - x
- deltay = int(xy[1]) - y
-
- distances[termDocs.doc()] = sqrt(deltax ** 2 +
- deltay ** 2)
-
- if not enumerator.next():
- break
- finally:
- termDocs.close()
+ def _getDistance(_self, doc):
- def compare(self, i, j):
+ deltax = _self.xDoc[doc] - self.x
+ deltay = _self.yDoc[doc] - self.y
- if self.distances[i.doc] < self.distances[j.doc]:
+ return sqrt(deltax * deltax + deltay * deltay)
+
+ def compare(_self, slot1, slot2):
+
+ if _self.values[slot1] < _self.values[slot2]:
return -1
- if self.distances[i.doc] > self.distances[j.doc]:
+ if _self.values[slot1] > _self.values[slot2]:
return 1
+
return 0
- def sortValue(self, i):
+ def setBottom(_self, slot):
+
+ _self._bottom = _self.values[slot]
+
+ def compareBottom(_self, doc):
+
+ docDistance = _self._getDistance(doc)
+ if _self._bottom < docDistance:
+ return -1
+ if _self._bottom > docDistance:
+ return 1
+
+ return 0
+
+ def copy(_self, slot, doc):
+
+ _self.values[slot] = _self._getDistance(doc)
- return Double(self.distances[i.doc])
+ def value(_self, slot):
- def sortType(self):
+ return Double(_self.values[slot])
- return SortField.FLOAT
+ def sortType(_self):
+ return SortField.CUSTOM
- return DistanceScoreDocLookupComparator(reader, fieldName,
- self.x, self.y)
+ return DistanceScoreDocLookupComparator(fieldName, numHits)
def __str__(self):
- return "Distance from ("+x+","+y+")"
+ return "Distance from (" + self.x + "," + self.y + ")"
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceSortingTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceSortingTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceSortingTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceSortingTest.py Wed Nov 25 04:08:13 2009
@@ -28,7 +28,8 @@
def setUp(self):
self.directory = RAMDirectory()
- writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
self.addPoint(writer, "El Charro", "restaurant", 1, 2)
self.addPoint(writer, "Cafe Poca Cosa", "restaurant", 5, 9)
@@ -37,25 +38,28 @@
writer.close()
- self.searcher = IndexSearcher(self.directory)
+ self.searcher = IndexSearcher(self.directory, True)
self.query = TermQuery(Term("type", "restaurant"))
def addPoint(self, writer, name, type, x, y):
doc = Document()
- doc.add(Field("name", name, Field.Store.YES, Field.Index.UN_TOKENIZED))
- doc.add(Field("type", type, Field.Store.YES, Field.Index.UN_TOKENIZED))
- doc.add(Field("location", "%d,%d" %(x, y),
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ doc.add(Field("name", name, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(Field("type", type, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(Field("x", str(x), Field.Store.YES,
+ Field.Index.NOT_ANALYZED_NO_NORMS))
+ doc.add(Field("y", str(y), Field.Store.YES,
+ Field.Index.NOT_ANALYZED_NO_NORMS));
+
writer.addDocument(doc)
def testNearestRestaurantToHome(self):
sort = Sort(SortField("location", DistanceComparatorSource(0, 0)))
- hits = self.searcher.search(self.query, sort)
- self.assertEqual("El Charro", hits.doc(0).get("name"), "closest")
- self.assertEqual("Los Betos", hits.doc(3).get("name"), "furthest")
+ scoreDocs = self.searcher.search(self.query, None, 50, sort).scoreDocs
+ self.assertEqual("El Charro", self.searcher.doc(scoreDocs[0].doc).get("name"), "closest")
+ self.assertEqual("Los Betos", self.searcher.doc(scoreDocs[3].doc).get("name"), "furthest")
def testNeareastRestaurantToWork(self):
@@ -67,8 +71,9 @@
fieldDoc = FieldDoc.cast_(docs.scoreDocs[0])
distance = Double.cast_(fieldDoc.fields[0]).doubleValue()
+
self.assertEqual(sqrt(17), distance,
- "(10,10) -> (9,6) = sqrt(17)")
+ "(10,10) -> (9,6) = sqrt(17)")
document = self.searcher.doc(fieldDoc.doc)
self.assertEqual("Los Betos", document["name"])
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/BaseIndexingTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/BaseIndexingTestCase.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/BaseIndexingTestCase.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/BaseIndexingTestCase.py Wed Nov 25 04:08:13 2009
@@ -17,11 +17,12 @@
from unittest import TestCase
from lucene import \
- FSDirectory, System, \
- Document, Field, SimpleAnalyzer, IndexWriter, IndexReader
+ SimpleFSDirectory, System, File, \
+ Document, Field, SimpleAnalyzer, IndexWriter, IndexReader
class BaseIndexingTestCase(TestCase):
+
keywords = ["1", "2"]
unindexed = ["Netherlands", "Italy"]
unstored = ["Amsterdam has lots of bridges",
@@ -32,24 +33,25 @@
indexDir = os.path.join(System.getProperty('java.io.tmpdir', 'tmp'),
'index-dir')
- self.dir = FSDirectory.getDirectory(indexDir, True)
+ self.dir = SimpleFSDirectory(File(indexDir))
self.addDocuments(self.dir)
def addDocuments(self, dir):
- writer = IndexWriter(dir, self.getAnalyzer(), True)
+ writer = IndexWriter(dir, self.getAnalyzer(), True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
writer.setUseCompoundFile(self.isCompound())
for i in xrange(len(self.keywords)):
doc = Document()
doc.add(Field("id", self.keywords[i],
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("country", self.unindexed[i],
Field.Store.YES, Field.Index.NO))
doc.add(Field("contents", self.unstored[i],
- Field.Store.NO, Field.Index.TOKENIZED))
+ Field.Store.NO, Field.Index.ANALYZED))
doc.add(Field("city", self.text[i],
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
@@ -65,13 +67,14 @@
def testIndexWriter(self):
- writer = IndexWriter(self.dir, self.getAnalyzer(), False)
- self.assertEqual(len(self.keywords), writer.docCount())
+ writer = IndexWriter(self.dir, self.getAnalyzer(), False,
+ IndexWriter.MaxFieldLength.UNLIMITED)
+ self.assertEqual(len(self.keywords), writer.numDocs())
writer.close()
def testIndexReader(self):
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, True)
self.assertEqual(len(self.keywords), reader.maxDoc())
self.assertEqual(len(self.keywords), reader.numDocs())
reader.close()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/CompoundVersusMultiFileIndexTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/CompoundVersusMultiFileIndexTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/CompoundVersusMultiFileIndexTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/CompoundVersusMultiFileIndexTest.py Wed Nov 25 04:08:13 2009
@@ -19,7 +19,8 @@
from datetime import timedelta
from lucene import \
- IndexWriter, SimpleAnalyzer, FSDirectory, System, Document, Field
+ IndexWriter, SimpleAnalyzer, SimpleFSDirectory, Document, Field, \
+ System, File
class CompoundVersusMultiFileIndexTest(TestCase):
@@ -39,8 +40,8 @@
self.rmdir(cIndexDir)
self.rmdir(mIndexDir)
- self.cDir = FSDirectory.getDirectory(cIndexDir, True)
- self.mDir = FSDirectory.getDirectory(mIndexDir, True)
+ self.cDir = SimpleFSDirectory(File(cIndexDir))
+ self.mDir = SimpleFSDirectory(File(mIndexDir))
def rmdir(self, dir):
@@ -69,7 +70,8 @@
def addDocuments(self, dir, isCompound):
- writer = IndexWriter(dir, SimpleAnalyzer(), True)
+ writer = IndexWriter(dir, SimpleAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
writer.setUseCompoundFile(isCompound)
# change to adjust performance of indexing with FSDirectory
@@ -80,13 +82,13 @@
for word in self.docs:
doc = Document()
doc.add(Field("keyword", word,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("unindexed", word,
Field.Store.YES, Field.Index.NO))
doc.add(Field("unstored", word,
- Field.Store.NO, Field.Index.TOKENIZED))
+ Field.Store.NO, Field.Index.ANALYZED))
doc.add(Field("text", word,
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentDeleteTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentDeleteTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentDeleteTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentDeleteTest.py Wed Nov 25 04:08:13 2009
@@ -20,7 +20,7 @@
def testDeleteBeforeIndexMerge(self):
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, False)
self.assertEqual(2, reader.maxDoc())
self.assertEqual(2, reader.numDocs())
reader.deleteDocument(1)
@@ -32,7 +32,7 @@
reader.close()
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, True)
self.assertEqual(2, reader.maxDoc())
self.assertEqual(1, reader.numDocs())
@@ -41,17 +41,18 @@
def testDeleteAfterIndexMerge(self):
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, False)
self.assertEqual(2, reader.maxDoc())
self.assertEqual(2, reader.numDocs())
reader.deleteDocument(1)
reader.close()
- writer = IndexWriter(self.dir, self.getAnalyzer(), False)
+ writer = IndexWriter(self.dir, self.getAnalyzer(), False,
+ IndexWriter.MaxFieldLength.UNLIMITED)
writer.optimize()
writer.close()
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, True)
self.assert_(not reader.isDeleted(1))
self.assert_(not reader.hasDeletions())
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentUpdateTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentUpdateTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentUpdateTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentUpdateTest.py Wed Nov 25 04:08:13 2009
@@ -25,19 +25,20 @@
self.assertEqual(1, self.getHitCount("city", "Amsterdam"))
- reader = IndexReader.open(self.dir)
+ reader = IndexReader.open(self.dir, False)
reader.deleteDocuments(Term("city", "Amsterdam"))
reader.close()
- writer = IndexWriter(self.dir, self.getAnalyzer(), False)
+ writer = IndexWriter(self.dir, self.getAnalyzer(), False,
+ IndexWriter.MaxFieldLength.UNLIMITED)
doc = Document()
- doc.add(Field("id", "1", Field.Store.YES, Field.Index.UN_TOKENIZED))
+ doc.add(Field("id", "1", Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("country", "Russia",
Field.Store.YES, Field.Index.NO))
doc.add(Field("contents", "St. Petersburg has lots of bridges",
- Field.Store.NO, Field.Index.TOKENIZED))
+ Field.Store.NO, Field.Index.ANALYZED))
doc.add(Field("city", "St. Petersburg",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
writer.close()
@@ -50,14 +51,12 @@
return WhitespaceAnalyzer()
-
def getHitCount(self, fieldName, searchString):
- searcher = IndexSearcher(self.dir)
+ searcher = IndexSearcher(self.dir, True)
t = Term(fieldName, searchString)
query = TermQuery(t)
- hits = searcher.search(query)
- hitCount = hits.length()
+ hitCount = len(searcher.search(query, 50).scoreDocs)
searcher.close()
return hitCount
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FS2RAMDirectoryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FS2RAMDirectoryTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FS2RAMDirectoryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FS2RAMDirectoryTest.py Wed Nov 25 04:08:13 2009
@@ -22,12 +22,12 @@
def testSlurp(self):
- fsDirReader = IndexReader.open(self.dir)
+ fsDirReader = IndexReader.open(self.dir, True)
self.assertEqual(len(self.keywords), fsDirReader.maxDoc())
self.assertEqual(len(self.keywords), fsDirReader.numDocs())
ramDir = RAMDirectory(self.dir)
- ramDirReader = IndexReader.open(ramDir)
+ ramDirReader = IndexReader.open(ramDir, True)
self.assertEqual(fsDirReader.maxDoc(), ramDirReader.maxDoc())
self.assertEqual(fsDirReader.numDocs(), ramDirReader.numDocs())
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FSversusRAMDirectoryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FSversusRAMDirectoryTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FSversusRAMDirectoryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FSversusRAMDirectoryTest.py Wed Nov 25 04:08:13 2009
@@ -19,8 +19,8 @@
from datetime import timedelta
from lucene import \
- IndexWriter, SimpleAnalyzer, Document, Field, System, \
- FSDirectory, RAMDirectory
+ IndexWriter, SimpleAnalyzer, Document, Field, System, File, \
+ SimpleFSDirectory, RAMDirectory
class FSversusRAMDirectoryTest(TestCase):
@@ -35,7 +35,7 @@
fsIndexDir = os.path.join(System.getProperty("java.io.tmpdir", "tmp"),
"fs-index")
self.ramDir = RAMDirectory()
- self.fsDir = FSDirectory.getDirectory(fsIndexDir, True)
+ self.fsDir = SimpleFSDirectory(File(fsIndexDir))
def testTiming(self):
@@ -56,7 +56,8 @@
def addDocuments(self, dir):
- writer = IndexWriter(dir, SimpleAnalyzer(), True)
+ writer = IndexWriter(dir, SimpleAnalyzer(), True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
#
# change to adjust performance of indexing with FSDirectory
@@ -68,13 +69,13 @@
for word in self.docs:
doc = Document()
doc.add(Field("keyword", word,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
doc.add(Field("unindexed", word,
Field.Store.YES, Field.Index.NO))
doc.add(Field("unstored", word,
- Field.Store.NO, Field.Index.TOKENIZED))
+ Field.Store.NO, Field.Index.ANALYZED))
doc.add(Field("text", word,
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FieldLengthTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FieldLengthTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FieldLengthTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FieldLengthTest.py Wed Nov 25 04:08:13 2009
@@ -14,13 +14,14 @@
import os
+from itertools import izip
from unittest import TestCase
from time import time
from datetime import timedelta
from lucene import \
- IndexWriter, SimpleAnalyzer, Document, Field, System, \
- Term, TermQuery, IndexSearcher, FSDirectory
+ IndexWriter, SimpleAnalyzer, Document, Field, System, File, \
+ Term, TermQuery, IndexSearcher, SimpleFSDirectory
class FieldLengthTest(TestCase):
@@ -35,7 +36,7 @@
indexDir = os.path.join(System.getProperty("java.io.tmpdir", "tmp"),
"index-dir")
- self.dir = FSDirectory.getDirectory(indexDir, True)
+ self.dir = SimpleFSDirectory(File(indexDir))
def testFieldSize(self):
@@ -47,30 +48,30 @@
def getHitCount(self, fieldName, searchString):
- searcher = IndexSearcher(self.dir)
+ searcher = IndexSearcher(self.dir, True)
t = Term(fieldName, searchString)
query = TermQuery(t)
- hits = searcher.search(query)
- hitCount = hits.length()
+ hitCount = len(searcher.search(query, 50).scoreDocs)
searcher.close()
return hitCount
def addDocuments(self, dir, maxFieldLength):
- writer = IndexWriter(dir, SimpleAnalyzer(), True)
- writer.setMaxFieldLength(maxFieldLength)
+ writer = IndexWriter(dir, SimpleAnalyzer(), True,
+ IndexWriter.MaxFieldLength(maxFieldLength))
- for i in xrange(len(self.keywords)):
+ for keyword, unindexed, unstored, text in \
+ izip(self.keywords, self.unindexed, self.unstored, self.text):
doc = Document()
- doc.add(Field("id", self.keywords[i],
- Field.Store.YES, Field.Index.UN_TOKENIZED))
- doc.add(Field("country", self.unindexed[i],
+ doc.add(Field("id", keyword,
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(Field("country", unindexed,
Field.Store.YES, Field.Index.NO))
- doc.add(Field("contents", self.unstored[i],
- Field.Store.NO, Field.Index.TOKENIZED))
- doc.add(Field("city", self.text[i],
- Field.Store.YES, Field.Index.TOKENIZED))
+ doc.add(Field("contents", unstored,
+ Field.Store.NO, Field.Index.ANALYZED))
+ doc.add(Field("city", text,
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/LockTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/LockTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/LockTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/LockTest.py Wed Nov 25 04:08:13 2009
@@ -17,7 +17,7 @@
from unittest import TestCase
from lucene import VERSION, \
- IndexWriter, IndexReader, SimpleAnalyzer, FSDirectory, System
+ IndexWriter, IndexReader, SimpleAnalyzer, SimpleFSDirectory, System, File
class LockTest(TestCase):
@@ -26,39 +26,22 @@
indexDir = os.path.join(System.getProperty("java.io.tmpdir", "tmp"),
"index")
- self.dir = FSDirectory.getDirectory(indexDir, True)
+ self.dir = SimpleFSDirectory(File(indexDir))
def testWriteLock(self):
- if VERSION < '2.1.0':
- writer1 = None
- writer2 = None
- gotException = False
-
- try:
- try:
- writer1 = IndexWriter(self.dir, SimpleAnalyzer(), True)
- writer2 = IndexWriter(self.dir, SimpleAnalyzer(), True)
-
- self.fail("We should never reach this point")
- except:
- gotException = True
- finally:
- writer1.close()
- self.assert_(writer2 is None)
- self.assert_(gotException)
-
- def testCommitLock(self):
-
- reader1 = None
- reader2 = None
+ writer1 = IndexWriter(self.dir, SimpleAnalyzer(),
+ IndexWriter.MaxFieldLength.UNLIMITED)
+ writer2 = None
try:
- writer = IndexWriter(self.dir, SimpleAnalyzer(), True)
- writer.close()
-
- reader1 = IndexReader.open(self.dir)
- reader2 = IndexReader.open(self.dir)
+ try:
+ writer2 = IndexWriter(self.dir, SimpleAnalyzer(),
+ IndexWriter.MaxFieldLength.UNLIMITED)
+ self.fail("We should never reach this point")
+ except:
+ pass
finally:
- reader1.close()
- reader2.close()
+ IndexWriter.unlock(self.dir)
+ writer1.close()
+ self.assert_(writer2 is None)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BasicSearchingTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BasicSearchingTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BasicSearchingTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BasicSearchingTest.py Wed Nov 25 04:08:13 2009
@@ -15,44 +15,46 @@
from lia.common.LiaTestCase import LiaTestCase
from lucene import \
- SimpleAnalyzer, Document, TermQuery, QueryParser, IndexSearcher, Term
+ SimpleAnalyzer, Document, TermQuery, QueryParser, IndexSearcher, Term, \
+ Version
class BasicSearchingTest(LiaTestCase):
def testTerm(self):
- searcher = IndexSearcher(self.directory)
+ searcher = IndexSearcher(self.directory, True)
t = Term("subject", "ant")
query = TermQuery(t)
- hits = searcher.search(query)
- self.assertEqual(1, hits.length(), "JDwA")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(1, len(scoreDocs), "JDwA")
t = Term("subject", "junit")
- hits = searcher.search(TermQuery(t))
- self.assertEqual(2, hits.length())
+ scoreDocs = searcher.search(TermQuery(t), 50).scoreDocs
+ self.assertEqual(2, len(scoreDocs))
searcher.close()
def testKeyword(self):
- searcher = IndexSearcher(self.directory)
+ searcher = IndexSearcher(self.directory, True)
t = Term("isbn", "1930110995")
query = TermQuery(t)
- hits = searcher.search(query)
- self.assertEqual(1, hits.length(), "JUnit in Action")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(1, len(scoreDocs), "JUnit in Action")
def testQueryParser(self):
- searcher = IndexSearcher(self.directory)
+ searcher = IndexSearcher(self.directory, True)
- query = QueryParser("contents",
+ query = QueryParser(Version.LUCENE_CURRENT, "contents",
SimpleAnalyzer()).parse("+JUNIT +ANT -MOCK")
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
- d = hits.doc(0)
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(1, len(scoreDocs))
+ d = searcher.doc(scoreDocs[0].doc)
self.assertEqual("Java Development with Ant", d.get("title"))
- query = QueryParser("contents", SimpleAnalyzer()).parse("mock OR junit")
- hits = searcher.search(query)
- self.assertEqual(2, hits.length(), "JDwA and JIA")
+ query = QueryParser(Version.LUCENE_CURRENT, "contents",
+ SimpleAnalyzer()).parse("mock OR junit")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(2, len(scoreDocs), "JDwA and JIA")
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BooleanQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BooleanQueryTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BooleanQueryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BooleanQueryTest.py Wed Nov 25 04:08:13 2009
@@ -14,8 +14,9 @@
from lia.common.LiaTestCase import LiaTestCase
-from lucene import\
- Term, BooleanQuery, IndexSearcher, RangeQuery, TermQuery, BooleanClause
+from lucene import Integer, \
+ Term, BooleanQuery, IndexSearcher, \
+ NumericRangeQuery, TermQuery, BooleanClause
class BooleanQueryTest(LiaTestCase):
@@ -23,17 +24,19 @@
def testAnd(self):
searchingBooks = TermQuery(Term("subject", "search"))
- currentBooks = RangeQuery(Term("pubmonth", "200401"),
- Term("pubmonth", "200412"), True)
+ books2004 = NumericRangeQuery.newIntRange("pubmonth",
+ Integer(200401),
+ Integer(200412),
+ True, True)
+
+ searchingBooks2004 = BooleanQuery()
+ searchingBooks2004.add(searchingBooks, BooleanClause.Occur.MUST)
+ searchingBooks2004.add(books2004, BooleanClause.Occur.MUST)
- currentSearchingBooks = BooleanQuery()
- currentSearchingBooks.add(searchingBooks, BooleanClause.Occur.MUST)
- currentSearchingBooks.add(currentBooks, BooleanClause.Occur.MUST)
+ searcher = IndexSearcher(self.directory, True)
+ scoreDocs = searcher.search(searchingBooks2004, 50).scoreDocs
- searcher = IndexSearcher(self.directory)
- hits = searcher.search(currentSearchingBooks)
-
- self.assertHitsIncludeTitle(hits, "Lucene in Action")
+ self.assertHitsIncludeTitle(searcher, scoreDocs, "Lucene in Action")
def testOr(self):
@@ -46,9 +49,11 @@
enlightenmentBooks.add(methodologyBooks, BooleanClause.Occur.SHOULD)
enlightenmentBooks.add(easternPhilosophyBooks, BooleanClause.Occur.SHOULD)
- searcher = IndexSearcher(self.directory)
- hits = searcher.search(enlightenmentBooks)
+ searcher = IndexSearcher(self.directory, True)
+ scoreDocs = searcher.search(enlightenmentBooks, 50).scoreDocs
print "or =", enlightenmentBooks
- self.assertHitsIncludeTitle(hits, "Extreme Programming Explained")
- self.assertHitsIncludeTitle(hits, u"Tao Te Ching \u9053\u5FB7\u7D93")
+ self.assertHitsIncludeTitle(searcher, scoreDocs,
+ "Extreme Programming Explained")
+ self.assertHitsIncludeTitle(searcher, scoreDocs,
+ u"Tao Te Ching \u9053\u5FB7\u7D93")
Added: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py?rev=883975&view=auto
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py (added)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py Wed Nov 25 04:08:13 2009
@@ -0,0 +1,46 @@
+# ====================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ====================================================================
+
+from lucene import Integer, \
+ IndexSearcher, NumericRangeQuery
+
+from lia.common.LiaTestCase import LiaTestCase
+
+
+class NumericRangeQueryTest(LiaTestCase):
+
+ def testInclusive(self):
+
+ searcher = IndexSearcher(self.directory, True)
+ # pub date of TTC was October 1988
+ query = NumericRangeQuery.newIntRange("pubmonth",
+ Integer(198805),
+ Integer(198810),
+ True, True)
+
+ topDocs = searcher.search(query, 100)
+ self.assertEqual(1, topDocs.totalHits)
+ searcher.close()
+
+ def testExclusive(self):
+
+ searcher = IndexSearcher(self.directory, True)
+ # pub date of TTC was October 1988
+ query = NumericRangeQuery.newIntRange("pubmonth",
+ Integer(198805),
+ Integer(198810),
+ False, False)
+ topDocs = searcher.search(query, 100)
+ self.assertEqual(0, topDocs.totalHits)
+ searcher.close()
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PhraseQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PhraseQueryTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PhraseQueryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PhraseQueryTest.py Wed Nov 25 04:08:13 2009
@@ -25,10 +25,11 @@
# set up sample document
directory = RAMDirectory()
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.UNLIMITED)
doc = Document()
doc.add(Field("field", "the quick brown fox jumped over the lazy dog",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.close()
@@ -42,9 +43,9 @@
for word in phrase:
query.add(Term("field", word))
- hits = self.searcher.search(query)
+ topDocs = self.searcher.search(query, 50)
- return len(hits) > 0
+ return topDocs.totalHits > 0
def testSlopComparison(self):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PrefixQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PrefixQueryTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PrefixQueryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PrefixQueryTest.py Wed Nov 25 04:08:13 2009
@@ -21,17 +21,17 @@
def testPrefix(self):
- searcher = IndexSearcher(self.directory)
+ searcher = IndexSearcher(self.directory, True)
# search for programming books, including subcategories
term = Term("category", "/technology/computers/programming")
query = PrefixQuery(term)
- hits = searcher.search(query)
- programmingAndBelow = hits.length()
+ topDocs = searcher.search(query, 50)
+ programmingAndBelow = topDocs.totalHits
# only programming books, not subcategories
- hits = searcher.search(TermQuery(term))
- justProgramming = hits.length()
+ topDocs = searcher.search(TermQuery(term), 50)
+ justProgramming = topDocs.totalHits
self.assert_(programmingAndBelow > justProgramming)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/QueryParserTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/QueryParserTest.py?rev=883975&r1=883974&r2=883975&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/QueryParserTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/QueryParserTest.py Wed Nov 25 04:08:13 2009
@@ -17,7 +17,7 @@
from lucene import \
WhitespaceAnalyzer, StandardAnalyzer, Term, QueryParser, Locale, \
BooleanQuery, FuzzyQuery, IndexSearcher, TermRangeQuery, TermQuery, \
- BooleanClause
+ BooleanClause, Version
class QueryParserTest(LiaTestCase):
@@ -26,7 +26,7 @@
super(QueryParserTest, self).setUp()
self.analyzer = WhitespaceAnalyzer()
- self.searcher = IndexSearcher(self.directory)
+ self.searcher = IndexSearcher(self.directory, True)
def testToString(self):
@@ -41,57 +41,60 @@
def testPrefixQuery(self):
- parser = QueryParser("category", StandardAnalyzer())
+ parser = QueryParser(Version.LUCENE_CURRENT, "category",
+ StandardAnalyzer(Version.LUCENE_CURRENT))
parser.setLowercaseExpandedTerms(False)
print parser.parse("/Computers/technology*").toString("category")
def testGrouping(self):
- query = QueryParser("subject", self.analyzer).parse("(agile OR extreme) AND methodology")
- hits = self.searcher.search(query)
+ query = QueryParser(Version.LUCENE_CURRENT, "subject",
+ self.analyzer).parse("(agile OR extreme) AND methodology")
+ scoreDocs = self.searcher.search(query, 50).scoreDocs
+
+ self.assertHitsIncludeTitle(self.searcher, scoreDocs,
+ "Extreme Programming Explained")
+ self.assertHitsIncludeTitle(self.searcher, scoreDocs,
+ "The Pragmatic Programmer")
- self.assertHitsIncludeTitle(hits, "Extreme Programming Explained")
- self.assertHitsIncludeTitle(hits, "The Pragmatic Programmer")
+ def testTermRangeQuery(self):
- def testRangeQuery(self):
-
- parser = QueryParser("subject", self.analyzer)
- parser.setUseOldRangeQuery(True)
-
- query = parser.parse("pubmonth:[200401 TO 200412]")
+ query = QueryParser(Version.LUCENE_CURRENT, "subject",
+ self.analyzer).parse("title2:[K TO N]")
self.assert_(TermRangeQuery.instance_(query))
- hits = self.searcher.search(query)
- self.assertHitsIncludeTitle(hits, "Lucene in Action")
+ scoreDocs = self.searcher.search(query, 10).scoreDocs
+ self.assertHitsIncludeTitle(self.searcher, scoreDocs, "Mindstorms")
- query = QueryParser("pubmonth",
- self.analyzer).parse("{200201 TO 200208}")
+ query = QueryParser(Version.LUCENE_CURRENT, "subject",
+ self.analyzer).parse("title2:{K TO Mindstorms}")
+ scoreDocs = self.searcher.search(query, 10).scoreDocs
+ self.assertHitsIncludeTitle(self.searcher, scoreDocs, "Mindstorms",
+ True)
- hits = self.searcher.search(query)
- self.assertEqual(0, hits.length(), "JDwA in 200208")
-
def testDateRangeQuery(self):
# locale diff between jre and gcj 1/1/04 -> 01/01/04
# expression = "modified:[1/1/04 TO 12/31/04]"
expression = "modified:[01/01/04 TO 12/31/04]"
- parser = QueryParser("subject", self.analyzer)
+ parser = QueryParser(Version.LUCENE_CURRENT, "subject", self.analyzer)
parser.setLocale(Locale.US)
query = parser.parse(expression)
print expression, "parsed to", query
- hits = self.searcher.search(query)
- self.assert_(hits.length() > 0)
+ topDocs = self.searcher.search(query, 50)
+ self.assert_(topDocs.totalHits > 0)
def testSlop(self):
- q = QueryParser("field", self.analyzer).parse('"exact phrase"')
+ q = QueryParser(Version.LUCENE_CURRENT, "field",
+ self.analyzer).parse('"exact phrase"')
self.assertEqual("\"exact phrase\"", q.toString("field"),
"zero slop")
- qp = QueryParser("field", self.analyzer)
+ qp = QueryParser(Version.LUCENE_CURRENT, "field", self.analyzer)
qp.setPhraseSlop(5)
q = qp.parse('"sloppy phrase"')
self.assertEqual("\"sloppy phrase\"~5", q.toString("field"),
@@ -99,19 +102,22 @@
def testPhraseQuery(self):
- q = QueryParser("field",
- StandardAnalyzer()).parse('"This is Some Phrase*"')
+ analyzer = StandardAnalyzer(Version.LUCENE_24)
+ q = QueryParser(Version.LUCENE_24, "field",
+ analyzer).parse('"This is Some Phrase*"')
self.assertEqual("\"some phrase\"", q.toString("field"), "analyzed")
- q = QueryParser("field", self.analyzer).parse('"term"')
+ q = QueryParser(Version.LUCENE_CURRENT, "field",
+ self.analyzer).parse('"term"')
self.assert_(TermQuery.instance_(q), "reduced to TermQuery")
def testLowercasing(self):
- q = QueryParser("field", self.analyzer).parse("PrefixQuery*")
+ q = QueryParser(Version.LUCENE_CURRENT, "field",
+ self.analyzer).parse("PrefixQuery*")
self.assertEqual("prefixquery*", q.toString("field"), "lowercased")
- qp = QueryParser("field", self.analyzer)
+ qp = QueryParser(Version.LUCENE_CURRENT, "field", self.analyzer)
qp.setLowercaseExpandedTerms(False)
q = qp.parse("PrefixQuery*")
self.assertEqual("PrefixQuery*", q.toString("field"), "not lowercased")
@@ -119,29 +125,25 @@
def testWildcard(self):
try:
- QueryParser("field", self.analyzer).parse("*xyz")
+ QueryParser(Version.LUCENE_CURRENT, "field",
+ self.analyzer).parse("*xyz")
self.fail("Leading wildcard character should not be allowed")
except:
self.assert_(True)
def testBoost(self):
- q = QueryParser("field", self.analyzer).parse("term^2")
+ q = QueryParser(Version.LUCENE_CURRENT, "field",
+ self.analyzer).parse("term^2")
self.assertEqual("term^2.0", q.toString("field"))
def testParseException(self):
try:
- QueryParser("contents", self.analyzer).parse("^&#")
+ QueryParser(Version.LUCENE_CURRENT, "contents",
+ self.analyzer).parse("^&#")
except:
# expression is invalid, as expected
self.assert_(True)
else:
self.fail("ParseException expected, but not thrown")
-
-# public void testStopWord() throws ParseException {
-# Query q = QueryParser.parse("the AND drag", "field",
-# StopAnalyzer())
-# // QueryParser fails on the previous line - this is a known
-# // issue
-# }