You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2009/10/13 09:38:11 UTC
svn commit: r824626 [1/2] - in /lucene/pylucene/trunk: ./
java/org/apache/pylucene/analysis/ java/org/apache/pylucene/search/
java/org/apache/pylucene/store/ test/
Author: vajda
Date: Tue Oct 13 07:38:10 2009
New Revision: 824626
URL: http://svn.apache.org/viewvc?rev=824626&view=rev
Log:
- unit tests ported to new API
- removed InstantiatedIndex contrib from default build
- with (partial) JCC Java generics support, a lot less downcasting is needed
Added:
lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py (with props)
lucene/pylucene/trunk/test/test_BinaryDocument.py (with props)
Removed:
lucene/pylucene/trunk/test/test_Compress.py
Modified:
lucene/pylucene/trunk/CHANGES
lucene/pylucene/trunk/Makefile
lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java
lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java
lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java
lucene/pylucene/trunk/test/test_Analyzers.py
lucene/pylucene/trunk/test/test_Binary.py
lucene/pylucene/trunk/test/test_BooleanOr.py
lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py
lucene/pylucene/trunk/test/test_BooleanQuery.py
lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
lucene/pylucene/trunk/test/test_DocBoost.py
lucene/pylucene/trunk/test/test_FilteredQuery.py
lucene/pylucene/trunk/test/test_FuzzyQuery.py
lucene/pylucene/trunk/test/test_Highlighter.py
lucene/pylucene/trunk/test/test_Not.py
lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py
lucene/pylucene/trunk/test/test_PhraseQuery.py
lucene/pylucene/trunk/test/test_PositionIncrement.py
lucene/pylucene/trunk/test/test_PrefixFilter.py
lucene/pylucene/trunk/test/test_PrefixQuery.py
lucene/pylucene/trunk/test/test_PyLucene.py
lucene/pylucene/trunk/test/test_PyLuceneThread.py
lucene/pylucene/trunk/test/test_PythonDirectory.py
lucene/pylucene/trunk/test/test_PythonQueryParser.py
lucene/pylucene/trunk/test/test_RangeFilter.py
lucene/pylucene/trunk/test/test_RangeQuery.py
lucene/pylucene/trunk/test/test_RegexQuery.py
lucene/pylucene/trunk/test/test_RewriteQuery.py
lucene/pylucene/trunk/test/test_RuntimeException.py
lucene/pylucene/trunk/test/test_Similarity.py
lucene/pylucene/trunk/test/test_Sort.py
lucene/pylucene/trunk/test/test_StopAnalyzer.py
lucene/pylucene/trunk/test/test_StopWords.py
lucene/pylucene/trunk/test/test_ThaiAnalyzer.py
lucene/pylucene/trunk/test/test_bug1564.py
lucene/pylucene/trunk/test/test_bug1763.py
lucene/pylucene/trunk/test/test_bug1842.py
Modified: lucene/pylucene/trunk/CHANGES
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/CHANGES?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/CHANGES (original)
+++ lucene/pylucene/trunk/CHANGES Tue Oct 13 07:38:10 2009
@@ -1,4 +1,11 @@
+Version 2.9.0 ->
+----------------------
+ - unit tests ported to new API
+ - removed InstantiatedIndex contrib from default build
+ - with (partial) JCC Java generics support, a lot less downcasting is needed
+ -
+
Version 2.4.1 -> 2.9.0
----------------------
- renamed the Highlighter's SpanScorer class to HighlighterSpanScorer
Modified: lucene/pylucene/trunk/Makefile
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/Makefile?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/Makefile (original)
+++ lucene/pylucene/trunk/Makefile Tue Oct 13 07:38:10 2009
@@ -14,10 +14,10 @@
# site-packages directory.
#
-VERSION=2.9.0-1
+VERSION=3.0.0-1
LUCENE_SVN_VER=HEAD
-LUCENE_VER=2.9.0
-LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/java/tags/lucene_2_9_0
+LUCENE_VER=3.0.0
+LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/java/trunk
PYLUCENE:=$(shell pwd)
LUCENE=lucene-java-$(LUCENE_VER)
@@ -48,6 +48,13 @@
#JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64
#NUM_FILES=2
+# Mac OS X 10.6 (MacPorts 1.8.0 64-bit Python 2.6, Java 1.6)
+#PREFIX_PYTHON=/opt/local
+#ANT=ant
+#PYTHON=$(PREFIX_PYTHON)/bin/python
+#JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64
+#NUM_FILES=2
+
# Mac OS X 10.6 (64-bit and 32-bit Python 2.6 together, Java 1.6)
#PREFIX_PYTHON=/usr
#ANT=ant
@@ -128,7 +135,6 @@
MEMORY_JAR=$(LUCENE)/build/contrib/memory/lucene-memory-$(LUCENE_VER).jar
REGEX_JAR=$(LUCENE)/build/contrib/regex/lucene-regex-$(LUCENE_VER).jar
QUERIES_JAR=$(LUCENE)/build/contrib/queries/lucene-queries-$(LUCENE_VER).jar
-INSTANTIATED_JAR=$(LUCENE)/build/contrib/instantiated/lucene-instantiated-$(LUCENE_VER).jar
EXTENSIONS_JAR=build/jar/extensions.jar
@@ -177,15 +183,12 @@
$(QUERIES_JAR): $(LUCENE_JAR)
cd $(LUCENE)/contrib/queries; $(ANT) -Dversion=$(LUCENE_VER)
-$(INSTANTIATED_JAR): $(LUCENE_JAR)
- cd $(LUCENE)/contrib/instantiated; $(ANT) -Dversion=$(LUCENE_VER)
-
$(EXTENSIONS_JAR): $(LUCENE_JAR)
$(ANT) -f extensions.xml -Dlucene.dir=$(LUCENE)
JARS=$(LUCENE_JAR) $(SNOWBALL_JAR) $(ANALYZERS_JAR) \
$(HIGHLIGHTER_JAR) $(MEMORY_JAR) \
- $(REGEX_JAR) $(QUERIES_JAR) $(INSTANTIATED_JAR) $(EXTENSIONS_JAR)
+ $(REGEX_JAR) $(QUERIES_JAR) $(EXTENSIONS_JAR)
jars: $(JARS)
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java Tue Oct 13 07:38:10 2009
@@ -46,6 +46,6 @@
}
public native void pythonDecRef();
- public native Token next()
+ public native boolean incrementToken()
throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java Tue Oct 13 07:38:10 2009
@@ -44,7 +44,13 @@
}
public native void pythonDecRef();
- public native Token next();
- public native void reset();
- public native void close();
+
+ public native boolean incrementToken()
+ throws IOException;
+ public native void end()
+ throws IOException;
+ public native void reset()
+ throws IOException;
+ public native void close()
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java Tue Oct 13 07:38:10 2009
@@ -47,6 +47,6 @@
public native void pythonDecRef();
- public native Token next()
+ public native boolean incrementToken()
throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java Tue Oct 13 07:38:10 2009
@@ -15,13 +15,12 @@
package org.apache.pylucene.search;
+import java.io.IOException;
+
import org.apache.lucene.search.Filter;
import org.apache.lucene.index.IndexReader;
-import java.util.BitSet;
+import org.apache.lucene.util.DocIdBitSet;
-/**
- * @author Andi Vajda
- */
public class PythonFilter extends Filter {
@@ -47,5 +46,6 @@
}
public native void pythonDecRef();
- public native BitSet bits(IndexReader reader);
+ public native DocIdBitSet getDocIdSet(IndexReader reader)
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java Tue Oct 13 07:38:10 2009
@@ -15,6 +15,8 @@
package org.apache.pylucene.store;
+import java.io.IOException;
+
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -45,20 +47,27 @@
}
public native void pythonDecRef();
- public native void close();
- public native IndexOutput createOutput(String name);
- public native void deleteFile(String name);
- public native boolean fileExists(String name);
- public native long fileLength(String name);
- public native long fileModified(String name);
- public native String[] list();
- public native Lock makeLock(String name);
- public native IndexInput openInput(String name);
- public native IndexInput openInput(String name, int bufferSize);
- public native void touchFile(String name);
- /* no longer implemented, deprecated in Lucene 2.1 */
- public void renameFile(String from, String to)
- {
- }
+ public native void close()
+ throws IOException;
+ public native IndexOutput createOutput(String name)
+ throws IOException;
+ public native void deleteFile(String name)
+ throws IOException;
+ public native boolean fileExists(String name)
+ throws IOException;
+ public native long fileLength(String name)
+ throws IOException;
+ public native long fileModified(String name)
+ throws IOException;
+ public native String[] listAll()
+ throws IOException;
+ public native IndexInput openInput(String name)
+ throws IOException;
+ public native IndexInput openInput(String name, int bufferSize)
+ throws IOException;
+ public native void touchFile(String name)
+ throws IOException;
+ public native void sync(String name)
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java Tue Oct 13 07:38:10 2009
@@ -15,8 +15,10 @@
package org.apache.pylucene.store;
+import java.io.IOException;
import org.apache.lucene.store.BufferedIndexInput;
+
public class PythonIndexInput extends BufferedIndexInput {
private long pythonObject;
@@ -40,14 +42,19 @@
pythonDecRef();
}
- public native Object clone();
- public native void close();
public native void pythonDecRef();
+
+ public native Object clone();
public native long length();
- public native byte[] readInternal(int length, long pos);
- public native void seekInternal(long pos);
+ public native void close()
+ throws IOException;
+ public native byte[] readInternal(int length, long pos)
+ throws IOException;
+ public native void seekInternal(long pos)
+ throws IOException;
protected void readInternal(byte[] b, int offset, int length)
+ throws IOException
{
byte[] data = readInternal(length, getFilePointer());
System.arraycopy(data, 0, b, offset, data.length);
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java Tue Oct 13 07:38:10 2009
@@ -50,12 +50,17 @@
}
public native void pythonDecRef();
- public native long length();
- public native void flushBuffer(byte[] data);
- public native void seekInternal(long pos);
- public native void close();
+ public native long length()
+ throws IOException;
+ public native void flushBuffer(byte[] data)
+ throws IOException;
+ public native void seekInternal(long pos)
+ throws IOException;
+ public native void close()
+ throws IOException;
protected void flushBuffer(byte[] b, int offset, int len)
+ throws IOException
{
byte[] data = new byte[len];
System.arraycopy(b, offset, data, 0, len);
Added: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py?rev=824626&view=auto
==============================================================================
--- lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py (added)
+++ lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py Tue Oct 13 07:38:10 2009
@@ -0,0 +1,104 @@
+# ====================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ====================================================================
+
+from unittest import TestCase, main
+from lucene import *
+
+class BaseTokenStreamTestCase(TestCase):
+ """
+ Base class for all Lucene unit tests that use TokenStreams.
+ """
+
+ def _assertTokenStreamContents(self, ts, output,
+ startOffsets=None, endOffsets=None,
+ types=None, posIncrements=None):
+
+ self.assert_(output is not None)
+ self.assert_(ts.hasAttribute(TermAttribute.class_),
+ "has TermAttribute")
+
+ termAtt = ts.getAttribute(TermAttribute.class_)
+
+ offsetAtt = None
+ if startOffsets is not None or endOffsets is not None:
+ self.assert_(ts.hasAttribute(OffsetAttribute.class_),
+ "has OffsetAttribute")
+ offsetAtt = ts.getAttribute(OffsetAttribute.class_)
+
+ typeAtt = None
+ if types is not None:
+ self.assert_(ts.hasAttribute(TypeAttribute.class_),
+ "has TypeAttribute")
+ typeAtt = ts.getAttribute(TypeAttribute.class_)
+
+ posIncrAtt = None
+ if posIncrements is not None:
+ self.assert_(ts.hasAttribute(PositionIncrementAttribute.class_),
+ "has PositionIncrementAttribute")
+ posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class_)
+
+ ts.reset()
+ for i in xrange(len(output)):
+ # extra safety to enforce, that the state is not preserved and
+ # also assign bogus values
+ ts.clearAttributes()
+ termAtt.setTermBuffer("bogusTerm")
+ if offsetAtt is not None:
+ offsetAtt.setOffset(14584724, 24683243)
+ if typeAtt is not None:
+ typeAtt.setType("bogusType")
+ if posIncrAtt is not None:
+ posIncrAtt.setPositionIncrement(45987657)
+
+ self.assert_(ts.incrementToken(), "token %d exists" %(i))
+ self.assertEqual(output[i], termAtt.term(), "term %d" %(i))
+ if startOffsets is not None:
+ self.assertEqual(startOffsets[i], offsetAtt.startOffset(),
+ "startOffset %d" %(i))
+ if endOffsets is not None:
+ self.assertEqual(endOffsets[i], offsetAtt.endOffset(),
+ "endOffset %d" %(i))
+ if types is not None:
+ self.assertEqual(types[i], typeAtt.type(), "type %d" %(i))
+ if posIncrements is not None:
+ self.assertEqual(posIncrements[i],
+ posIncrAtt.getPositionIncrement(),
+ "posIncrement %d" %(i))
+
+ self.assert_(not ts.incrementToken(), "end of stream")
+ ts.end()
+ ts.close()
+
+ def _assertAnalyzesTo(self, a, input, output,
+ startOffsets=None, endOffsets=None,
+ types=None, posIncrements=None):
+
+ ts = a.tokenStream("dummy", StringReader(input))
+ self._assertTokenStreamContents(ts, output, startOffsets, endOffsets,
+ types, posIncrements)
+
+ def _assertAnalyzesToReuse(self, a, input, output,
+ startOffsets=None, endOffsets=None,
+ types=None, posIncrements=None):
+
+ ts = a.reusableTokenStream("dummy", StringReader(input))
+ self._assertTokenStreamContents(ts, output, startOffsets, endOffsets,
+ types, posIncrements)
+
+ # simple utility method for testing stemmers
+ def _checkOneTerm(self, a, input, expected):
+ self._assertAnalyzesTo(a, input, JArray('string')(expected))
+
+ def _checkOneTermReuse(self, a, input, expected):
+ self._assertAnalyzesToReuse(a, input, JArray('string')(expected))
Propchange: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: lucene/pylucene/trunk/test/test_Analyzers.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Analyzers.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Analyzers.py (original)
+++ lucene/pylucene/trunk/test/test_Analyzers.py Tue Oct 13 07:38:10 2009
@@ -12,27 +12,16 @@
# limitations under the License.
# ====================================================================
-from unittest import TestCase, main
+from unittest import main
+from BaseTokenStreamTestCase import BaseTokenStreamTestCase;
from lucene import *
-class AnalyzersTestCase(TestCase):
+class AnalyzersTestCase(BaseTokenStreamTestCase):
"""
Unit tests ported from Java Lucene
"""
- def _assertAnalyzesTo(self, a, input, output):
-
- ts = a.tokenStream("dummy", StringReader(input))
- for string in output:
- t = ts.next()
- self.assert_(t is not None)
- self.assertEqual(t.termText(), string)
-
- self.assert_(not list(ts))
- ts.close()
-
-
def testSimple(self):
a = SimpleAnalyzer()
@@ -53,6 +42,7 @@
self._assertAnalyzesTo(a, "\"QUOTED\" word",
[ "quoted", "word" ])
+
def testNull(self):
a = WhitespaceAnalyzer()
@@ -81,10 +71,53 @@
self._assertAnalyzesTo(a, "foo a bar such FOO THESE BAR",
[ "foo", "bar", "foo", "bar" ])
+ def _verifyPayload(self, ts):
+
+ payloadAtt = ts.getAttribute(PayloadAttribute.class_)
+ b = 0
+ while True:
+ b += 1
+ if not ts.incrementToken():
+ break
+ self.assertEqual(b, payloadAtt.getPayload().toByteArray()[0])
+
+ # Make sure old style next() calls result in a new copy of payloads
+ def testPayloadCopy(self):
+
+ s = "how now brown cow"
+ ts = WhitespaceTokenizer(StringReader(s))
+ ts = PayloadSetter(ts)
+ self._verifyPayload(ts)
+
+ ts = WhitespaceTokenizer(StringReader(s))
+ ts = PayloadSetter(ts)
+ self._verifyPayload(ts)
+
+
+class PayloadSetter(PythonTokenFilter):
+
+ def __init__(self, input):
+ super(PayloadSetter, self).__init__(input)
+
+ self.input = input
+ self.payloadAtt = self.addAttribute(PayloadAttribute.class_)
+ self.data = JArray('byte')(1)
+ self.p = Payload(self.data, 0, 1)
+
+ def incrementToken(self):
+
+ if not self.input.incrementToken():
+ return False
+
+ self.payloadAtt.setPayload(self.p)
+ self.data[0] += 1;
+
+ return True
+
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_Binary.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Binary.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Binary.py (original)
+++ lucene/pylucene/trunk/test/test_Binary.py Tue Oct 13 07:38:10 2009
@@ -21,22 +21,22 @@
def binary(self, b):
- c = JArray('byte')([chr(a) for a in b])
+ c = JArray('byte')(b)
field = Field("bin", c, Field.Store.YES)
- v = field.binaryValue()
- assert c == v and b == [ord(a) for a in v]
+ v = field.binaryValue
+ assert c == v and b == [a for a in v]
def testBinary(self):
self.binary([66, 90, 104, 57, 49, 65, 89, 38,
- 83, 89, 105, 56, 95, 75, 0, 0, 14, 215, 128])
+ 83, 89, 105, 56, 95, 75, 0, 0, 14, -41, -128])
self.binary([])
self.binary([0, 0, 0])
if __name__ == '__main__':
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Added: lucene/pylucene/trunk/test/test_BinaryDocument.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BinaryDocument.py?rev=824626&view=auto
==============================================================================
--- lucene/pylucene/trunk/test/test_BinaryDocument.py (added)
+++ lucene/pylucene/trunk/test/test_BinaryDocument.py Tue Oct 13 07:38:10 2009
@@ -0,0 +1,120 @@
+# ====================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ====================================================================
+
+from unittest import TestCase, main
+from lucene import *
+
+
+class TestBinaryDocument(TestCase):
+
+ binaryValStored = "this text will be stored as a byte array in the index"
+ binaryValCompressed = "this text will be also stored and compressed as a byte array in the index"
+
+ def testBinaryFieldInIndex(self):
+
+ bytes = JArray('byte')(self.binaryValStored)
+ binaryFldStored = Field("binaryStored", bytes,
+ Field.Store.YES)
+ stringFldStored = Field("stringStored", self.binaryValStored,
+ Field.Store.YES, Field.Index.NO,
+ Field.TermVector.NO)
+
+ try:
+ # binary fields with store off are not allowed
+ Field("fail", bytes, Field.Store.NO)
+ self.fail()
+ except JavaError, e:
+ self.assertEqual(e.getJavaException().getClass().getName(),
+ 'java.lang.IllegalArgumentException')
+
+ doc = Document()
+ doc.add(binaryFldStored)
+ doc.add(stringFldStored)
+
+ # test for field count
+ self.assertEqual(2, doc.fields.size())
+
+ # add the doc to a ram index
+ dir = RAMDirectory()
+ writer = IndexWriter(dir, StandardAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
+ writer.addDocument(doc)
+ writer.close()
+
+ # open a reader and fetch the document
+ reader = IndexReader.open(dir, False)
+ docFromReader = reader.document(0)
+ self.assert_(docFromReader is not None)
+
+ # fetch the binary stored field and compare it's content with the
+ # original one
+ binaryFldStoredTest = str(String(docFromReader.getBinaryValue("binaryStored")))
+ self.assertEqual(binaryFldStoredTest, self.binaryValStored)
+
+ # fetch the string field and compare it's content with the original
+ # one
+ stringFldStoredTest = docFromReader.get("stringStored")
+ self.assertEqual(stringFldStoredTest, self.binaryValStored)
+
+ # delete the document from index
+ reader.deleteDocument(0)
+ self.assertEqual(0, reader.numDocs())
+
+ reader.close()
+ dir.close()
+
+ def testCompressionTools(self):
+
+ bytes = JArray('byte')(self.binaryValCompressed)
+ binaryFldCompressed = Field("binaryCompressed", CompressionTools.compress(bytes), Field.Store.YES)
+ stringFldCompressed = Field("stringCompressed", CompressionTools.compressString(self.binaryValCompressed), Field.Store.YES)
+
+ doc = Document()
+ doc.add(binaryFldCompressed)
+ doc.add(stringFldCompressed)
+
+ # add the doc to a ram index
+ dir = RAMDirectory()
+ writer = IndexWriter(dir, StandardAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
+ writer.addDocument(doc)
+ writer.close()
+
+ # open a reader and fetch the document
+ reader = IndexReader.open(dir, False)
+ docFromReader = reader.document(0)
+ self.assert_(docFromReader is not None)
+
+ # fetch the binary compressed field and compare it's content with
+ # the original one
+ binaryFldCompressedTest = str(String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed"))))
+ self.assertEqual(binaryFldCompressedTest, self.binaryValCompressed)
+ self.assertEqual(CompressionTools.decompressString(docFromReader.getBinaryValue("stringCompressed")), self.binaryValCompressed)
+
+ reader.close()
+ dir.close()
+
+
+if __name__ == '__main__':
+ import sys, lucene
+ lucene.initVM()
+ if '-loop' in sys.argv:
+ sys.argv.remove('-loop')
+ while True:
+ try:
+ main()
+ except:
+ pass
+ else:
+ main()
Propchange: lucene/pylucene/trunk/test/test_BinaryDocument.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/pylucene/trunk/test/test_BinaryDocument.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: lucene/pylucene/trunk/test/test_BooleanOr.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanOr.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanOr.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanOr.py Tue Oct 13 07:38:10 2009
@@ -38,23 +38,24 @@
def setUp(self):
rd = RAMDirectory()
- writer = IndexWriter(rd, StandardAnalyzer(), True)
+ writer = IndexWriter(rd, StandardAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
d = Document()
d.add(Field(self.FIELD_T,
"Optimize not deleting all files",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
d.add(Field(self.FIELD_C,
"Deleted When I run an optimize in our production environment.",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(d)
writer.close()
- self.searcher = IndexSearcher(rd)
+ self.searcher = IndexSearcher(rd, True)
def search(self, q):
- return self.searcher.search(q).length()
+ return self.searcher.search(q, 50).totalHits
def testElements(self):
@@ -114,7 +115,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py Tue Oct 13 07:38:10 2009
@@ -52,7 +52,7 @@
writer.close()
- reader = IndexReader.open(directory)
+ reader = IndexReader.open(directory, True)
query = PrefixQuery(Term("category", "foo"))
rw1 = query.rewrite(reader)
@@ -69,7 +69,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_BooleanQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanQuery.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanQuery.py Tue Oct 13 07:38:10 2009
@@ -46,7 +46,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_CachingWrapperFilter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_CachingWrapperFilter.py (original)
+++ lucene/pylucene/trunk/test/test_CachingWrapperFilter.py Tue Oct 13 07:38:10 2009
@@ -24,18 +24,19 @@
def testCachingWorks(self):
dir = RAMDirectory()
- writer = IndexWriter(dir, StandardAnalyzer(), True)
+ writer = IndexWriter(dir, StandardAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
writer.close()
- reader = IndexReader.open(dir)
+ reader = IndexReader.open(dir, True)
class mockFilter(PythonFilter):
def __init__(self):
super(mockFilter, self).__init__()
self._wasCalled = False
- def bits(self, reader):
+ def getDocIdSet(self, reader):
self._wasCalled = True;
- return BitSet()
+ return DocIdBitSet(BitSet())
def clear(self):
self._wasCalled = False
def wasCalled(self):
@@ -45,12 +46,12 @@
cacher = CachingWrapperFilter(filter)
# first time, nested filter is called
- cacher.bits(reader)
+ cacher.getDocIdSet(reader)
self.assert_(filter.wasCalled(), "first time")
# second time, nested filter should not be called
filter.clear()
- cacher.bits(reader)
+ cacher.getDocIdSet(reader)
self.assert_(not filter.wasCalled(), "second time")
reader.close()
@@ -58,7 +59,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_DocBoost.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_DocBoost.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_DocBoost.py (original)
+++ lucene/pylucene/trunk/test/test_DocBoost.py Tue Oct 13 07:38:10 2009
@@ -24,10 +24,11 @@
def testDocBoost(self):
store = RAMDirectory()
- writer = IndexWriter(store, SimpleAnalyzer(), True)
+ writer = IndexWriter(store, SimpleAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
- f1 = Field("field", "word", Field.Store.YES, Field.Index.TOKENIZED)
- f2 = Field("field", "word", Field.Store.YES, Field.Index.TOKENIZED)
+ f1 = Field("field", "word", Field.Store.YES, Field.Index.ANALYZED)
+ f2 = Field("field", "word", Field.Store.YES, Field.Index.ANALYZED)
f2.setBoost(2.0)
d1 = Document()
@@ -55,8 +56,8 @@
def collect(self, doc, score):
scores[doc] = score
- IndexSearcher(store).search(TermQuery(Term("field", "word")),
- hitCollector())
+ IndexSearcher(store, True).search(TermQuery(Term("field", "word")),
+ hitCollector())
lastScore = 0.0
for score in scores:
@@ -66,7 +67,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_FilteredQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_FilteredQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_FilteredQuery.py (original)
+++ lucene/pylucene/trunk/test/test_FilteredQuery.py Tue Oct 13 07:38:10 2009
@@ -24,52 +24,53 @@
def setUp(self):
self.directory = RAMDirectory()
- writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
doc = Document()
doc.add(Field("field", "one two three four five",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
doc.add(Field("sorter", "b",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
doc = Document()
doc.add(Field("field", "one two three four",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
doc.add(Field("sorter", "d",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
doc = Document()
doc.add(Field("field", "one two three y",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
doc.add(Field("sorter", "a",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
doc = Document()
doc.add(Field("field", "one two x",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
doc.add(Field("sorter", "c",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
writer.close()
- self.searcher = IndexSearcher(self.directory)
+ self.searcher = IndexSearcher(self.directory, True)
self.query = TermQuery(Term("field", "three"))
class filter(PythonFilter):
- def bits(self, reader):
+ def getDocIdSet(self, reader):
bitset = BitSet(5)
bitset.set(1)
bitset.set(3)
- return bitset
+ return DocIdBitSet(bitset)
self.filter = filter()
@@ -81,44 +82,44 @@
def testFilteredQuery(self):
filteredquery = FilteredQuery(self.query, self.filter)
- hits = self.searcher.search(filteredquery);
- self.assertEqual(1, hits.length())
- self.assertEqual(1, hits.id(0))
-
- hits = self.searcher.search(filteredquery, Sort("sorter"))
- self.assertEqual(1, hits.length())
- self.assertEqual(1, hits.id(0))
+ topDocs = self.searcher.search(filteredquery, 50)
+ self.assertEqual(1, topDocs.totalHits)
+ self.assertEqual(1, topDocs.scoreDocs[0].doc)
+
+ topDocs = self.searcher.search(filteredquery, None, 50, Sort("sorter"))
+ self.assertEqual(1, topDocs.totalHits)
+ self.assertEqual(1, topDocs.scoreDocs[0].doc)
filteredquery = FilteredQuery(TermQuery(Term("field", "one")),
self.filter)
- hits = self.searcher.search(filteredquery)
- self.assertEqual(2, hits.length())
+ topDocs = self.searcher.search(filteredquery, 50)
+ self.assertEqual(2, topDocs.totalHits)
filteredquery = FilteredQuery(TermQuery(Term("field", "x")),
self.filter)
- hits = self.searcher.search(filteredquery)
- self.assertEqual(1, hits.length())
- self.assertEqual(3, hits.id(0))
+ topDocs = self.searcher.search(filteredquery, 50)
+ self.assertEqual(1, topDocs.totalHits)
+ self.assertEqual(3, topDocs.scoreDocs[0].doc)
filteredquery = FilteredQuery(TermQuery(Term("field", "y")),
self.filter)
- hits = self.searcher.search(filteredquery)
- self.assertEqual(0, hits.length())
+ topDocs = self.searcher.search(filteredquery, 50)
+ self.assertEqual(0, topDocs.totalHits)
def testRangeQuery(self):
"""
This tests FilteredQuery's rewrite correctness
"""
-
- rq = RangeQuery(Term("sorter", "b"), Term("sorter", "d"), True)
+
+ rq = TermRangeQuery("sorter", "b", "d", True, True)
filteredquery = FilteredQuery(rq, self.filter)
- hits = self.searcher.search(filteredquery)
- self.assertEqual(2, hits.length())
+ scoreDocs = self.searcher.search(filteredquery, 1000).scoreDocs
+ self.assertEqual(2, len(scoreDocs))
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_FuzzyQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_FuzzyQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_FuzzyQuery.py (original)
+++ lucene/pylucene/trunk/test/test_FuzzyQuery.py Tue Oct 13 07:38:10 2009
@@ -22,15 +22,17 @@
"""
def _addDoc(self, text, writer):
+
doc = Document()
doc.add(Field("field", text,
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
def testDefaultFuzziness(self):
directory = RAMDirectory()
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
self._addDoc("aaaaa", writer)
self._addDoc("aaaab", writer)
self._addDoc("aaabb", writer)
@@ -41,47 +43,47 @@
writer.optimize()
writer.close()
- searcher = IndexSearcher(directory)
+ searcher = IndexSearcher(directory, True)
query = FuzzyQuery(Term("field", "aaaaa"))
- hits = searcher.search(query)
- self.assertEqual(3, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(3, topDocs.totalHits)
# not similar enough:
query = FuzzyQuery(Term("field", "xxxxx"))
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
# edit distance to "aaaaa" = 3
query = FuzzyQuery(Term("field", "aaccc"))
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
# query identical to a word in the index:
query = FuzzyQuery(Term("field", "aaaaa"))
- hits = searcher.search(query)
- self.assertEqual(3, hits.length())
- self.assertEqual(hits.doc(0).get("field"), "aaaaa")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(3, len(scoreDocs))
+ self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaa")
# default allows for up to two edits:
- self.assertEqual(hits.doc(1).get("field"), "aaaab")
- self.assertEqual(hits.doc(2).get("field"), "aaabb")
+ self.assertEqual(searcher.doc(scoreDocs[1].doc).get("field"), "aaaab")
+ self.assertEqual(searcher.doc(scoreDocs[2].doc).get("field"), "aaabb")
# query similar to a word in the index:
query = FuzzyQuery(Term("field", "aaaac"))
- hits = searcher.search(query)
- self.assertEqual(3, hits.length())
- self.assertEqual(hits.doc(0).get("field"), "aaaaa")
- self.assertEqual(hits.doc(1).get("field"), "aaaab")
- self.assertEqual(hits.doc(2).get("field"), "aaabb")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(3, len(scoreDocs))
+ self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaa")
+ self.assertEqual(searcher.doc(scoreDocs[1].doc).get("field"), "aaaab")
+ self.assertEqual(searcher.doc(scoreDocs[2].doc).get("field"), "aaabb")
query = FuzzyQuery(Term("field", "ddddX"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
- self.assertEqual(hits.doc(0).get("field"), "ddddd")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(1, len(scoreDocs))
+ self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "ddddd")
# different field = no match:
query = FuzzyQuery(Term("anotherfield", "ddddX"))
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
searcher.close()
directory.close()
@@ -89,37 +91,38 @@
def testDefaultFuzzinessLong(self):
directory = RAMDirectory()
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
self._addDoc("aaaaaaa", writer)
self._addDoc("segment", writer)
writer.optimize()
writer.close()
- searcher = IndexSearcher(directory)
+ searcher = IndexSearcher(directory, True)
# not similar enough:
query = FuzzyQuery(Term("field", "xxxxx"))
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
# edit distance to "aaaaaaa" = 3, this matches because
# the string is longer than
# in testDefaultFuzziness so a bigger difference is allowed:
query = FuzzyQuery(Term("field", "aaaaccc"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
- self.assertEqual(hits.doc(0).get("field"), "aaaaaaa")
+ scoreDocs = searcher.search(query, 50).scoreDocs
+ self.assertEqual(1, len(scoreDocs))
+ self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaaaa")
# no match, more than half of the characters is wrong:
query = FuzzyQuery(Term("field", "aaacccc"))
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
# "student" and "stellent" are indeed similar to "segment" by default:
query = FuzzyQuery(Term("field", "student"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits)
query = FuzzyQuery(Term("field", "stellent"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits)
searcher.close()
directory.close()
@@ -127,7 +130,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_Highlighter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Highlighter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Highlighter.py (original)
+++ lucene/pylucene/trunk/test/test_Highlighter.py Tue Oct 13 07:38:10 2009
@@ -56,8 +56,8 @@
highlighter.setTextFragmenter(SimpleFragmenter(40))
maxNumFragmentsRequired = 2
- for i in range(0, self.hits.length()):
- text = self.hits.doc(i).get(self.FIELD_NAME)
+ for scoreDoc in self.scoreDocs:
+ text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
StringReader(text))
@@ -79,14 +79,14 @@
def doSearching(self, queryString):
- searcher = IndexSearcher(self.ramDir)
+ self.searcher = IndexSearcher(self.ramDir, True)
self.query = self.parser.parse(queryString)
# for any multi-term queries to work (prefix, wildcard, range,
# fuzzy etc) you must use a rewritten query!
self.query = self.query.rewrite(self.reader)
print "Searching for:", self.query.toString(self.FIELD_NAME)
- self.hits = searcher.search(self.query)
+ self.scoreDocs = self.searcher.search(self.query, 100).scoreDocs
self.numHighlights = 0
def doStandardHighlights(self):
@@ -95,8 +95,8 @@
highlighter = Highlighter(formatter, QueryScorer(self.query))
highlighter.setTextFragmenter(SimpleFragmenter(20))
- for i in range(0, self.hits.length()):
- text = self.hits.doc(i).get(self.FIELD_NAME)
+ for scoreDoc in self.scoreDocs:
+ text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
maxNumFragmentsRequired = 2
fragmentSeparator = "..."
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
@@ -114,22 +114,23 @@
def setUp(self):
- self.analyzer=StandardAnalyzer()
+ self.analyzer = StandardAnalyzer()
self.ramDir = RAMDirectory()
- writer = IndexWriter(self.ramDir, self.analyzer, True)
+ writer = IndexWriter(self.ramDir, self.analyzer, True,
+ IndexWriter.MaxFieldLength.LIMITED)
for text in self.texts:
self.addDoc(writer, text)
writer.optimize()
writer.close()
- self.reader = IndexReader.open(self.ramDir)
+ self.reader = IndexReader.open(self.ramDir, True)
self.numHighlights = 0;
def addDoc(self, writer, text):
d = Document()
f = Field(self.FIELD_NAME, text,
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES)
d.add(f)
writer.addDocument(d)
@@ -137,7 +138,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_Not.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Not.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Not.py (original)
+++ lucene/pylucene/trunk/test/test_Not.py Tue Oct 13 07:38:10 2009
@@ -24,25 +24,26 @@
def testNot(self):
store = RAMDirectory()
- writer = IndexWriter(store, SimpleAnalyzer(), True)
+ writer = IndexWriter(store, SimpleAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
d1 = Document()
- d1.add(Field("field", "a b", Field.Store.YES, Field.Index.TOKENIZED))
+ d1.add(Field("field", "a b", Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(d1)
writer.optimize()
writer.close()
- searcher = IndexSearcher(store)
+ searcher = IndexSearcher(store, True)
query = QueryParser("field", SimpleAnalyzer()).parse("a NOT b")
- hits = searcher.search(query)
- self.assertEqual(0, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits)
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py (original)
+++ lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py Tue Oct 13 07:38:10 2009
@@ -28,19 +28,22 @@
analyzer.addAnalyzer("special", SimpleAnalyzer())
tokenStream = analyzer.tokenStream("field", StringReader(text))
- token = tokenStream.next()
- self.assertEqual("Qwerty", token.termText(),
+ termAtt = tokenStream.getAttribute(TermAttribute.class_)
+
+ self.assert_(tokenStream.incrementToken())
+ self.assertEqual("Qwerty", termAtt.term(),
"WhitespaceAnalyzer does not lowercase")
tokenStream = analyzer.tokenStream("special", StringReader(text))
- token = tokenStream.next()
- self.assertEqual("qwerty", token.termText(),
+ termAtt = tokenStream.getAttribute(TermAttribute.class_)
+ self.assert_(tokenStream.incrementToken())
+ self.assertEqual("qwerty", termAtt.term(),
"SimpleAnalyzer lowercases")
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_PhraseQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PhraseQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PhraseQuery.py (original)
+++ lucene/pylucene/trunk/test/test_PhraseQuery.py Tue Oct 13 07:38:10 2009
@@ -24,17 +24,18 @@
def setUp(self):
self.directory = RAMDirectory()
- writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
doc = Document()
doc.add(Field("field", "one two three four five",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
writer.close()
- self.searcher = IndexSearcher(self.directory)
+ self.searcher = IndexSearcher(self.directory, True)
self.query = PhraseQuery()
def tearDown(self):
@@ -47,16 +48,16 @@
self.query.setSlop(2)
self.query.add(Term("field", "one"))
self.query.add(Term("field", "five"))
- hits = self.searcher.search(self.query)
- self.assertEqual(0, hits.length())
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(0, topDocs.totalHits)
def testBarelyCloseEnough(self):
self.query.setSlop(3)
self.query.add(Term("field", "one"))
self.query.add(Term("field", "five"))
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length())
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits)
def testExact(self):
"""
@@ -66,14 +67,14 @@
# slop is zero by default
self.query.add(Term("field", "four"))
self.query.add(Term("field", "five"))
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length(), "exact match")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits, "exact match")
self.query = PhraseQuery()
self.query.add(Term("field", "two"))
self.query.add(Term("field", "one"))
- hits = self.searcher.search(self.query)
- self.assertEqual(0, hits.length(), "reverse not exact")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(0, topDocs.totalHits, "reverse not exact")
def testSlop1(self):
@@ -81,8 +82,8 @@
self.query.setSlop(1)
self.query.add(Term("field", "one"))
self.query.add(Term("field", "two"))
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length(), "in order")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits, "in order")
# Ensures slop of 1 does not work for phrases out of order
# must be at least 2.
@@ -90,8 +91,8 @@
self.query.setSlop(1)
self.query.add(Term("field", "two"))
self.query.add(Term("field", "one"))
- hits = self.searcher.search(self.query)
- self.assertEqual(0, hits.length(), "reversed, slop not 2 or more")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(0, topDocs.totalHits, "reversed, slop not 2 or more")
def testOrderDoesntMatter(self):
"""
@@ -101,15 +102,15 @@
self.query.setSlop(2) # must be at least two for reverse order match
self.query.add(Term("field", "two"))
self.query.add(Term("field", "one"))
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length(), "just sloppy enough")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits, "just sloppy enough")
self.query = PhraseQuery()
self.query.setSlop(2)
self.query.add(Term("field", "three"))
self.query.add(Term("field", "one"))
- hits = self.searcher.search(self.query)
- self.assertEqual(0, hits.length(), "not sloppy enough")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(0, topDocs.totalHits, "not sloppy enough")
def testMulipleTerms(self):
"""
@@ -121,135 +122,138 @@
self.query.add(Term("field", "one"))
self.query.add(Term("field", "three"))
self.query.add(Term("field", "five"))
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length(), "two total moves")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits, "two total moves")
self.query = PhraseQuery()
self.query.setSlop(5) # it takes six moves to match this phrase
self.query.add(Term("field", "five"))
self.query.add(Term("field", "three"))
self.query.add(Term("field", "one"))
- hits = self.searcher.search(self.query)
- self.assertEqual(0, hits.length(), "slop of 5 not close enough")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(0, topDocs.totalHits, "slop of 5 not close enough")
self.query.setSlop(6)
- hits = self.searcher.search(self.query)
- self.assertEqual(1, hits.length(), "slop of 6 just right")
+ topDocs = self.searcher.search(self.query, 50)
+ self.assertEqual(1, topDocs.totalHits, "slop of 6 just right")
def testPhraseQueryWithStopAnalyzer(self):
directory = RAMDirectory()
stopAnalyzer = StopAnalyzer()
- writer = IndexWriter(directory, stopAnalyzer, True)
+ writer = IndexWriter(directory, stopAnalyzer, True,
+ IndexWriter.MaxFieldLength.LIMITED)
doc = Document()
doc.add(Field("field", "the stop words are here",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.close()
- searcher = IndexSearcher(directory)
+ searcher = IndexSearcher(directory, True)
# valid exact phrase query
query = PhraseQuery()
query.add(Term("field","stop"))
query.add(Term("field","words"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits)
# currently StopAnalyzer does not leave "holes", so this matches.
query = PhraseQuery()
query.add(Term("field", "words"))
query.add(Term("field", "here"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length())
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits)
searcher.close()
def testPhraseQueryInConjunctionScorer(self):
directory = RAMDirectory()
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
doc = Document()
doc.add(Field("source", "marketing info",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
writer.addDocument(doc)
doc = Document()
doc.add(Field("contents", "foobar",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
doc.add(Field("source", "marketing info",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
writer.addDocument(doc)
writer.optimize()
writer.close()
- searcher = IndexSearcher(directory)
+ searcher = IndexSearcher(directory, True)
phraseQuery = PhraseQuery()
phraseQuery.add(Term("source", "marketing"))
phraseQuery.add(Term("source", "info"))
- hits = searcher.search(phraseQuery)
- self.assertEqual(2, hits.length())
+ topDocs = searcher.search(phraseQuery, 50)
+ self.assertEqual(2, topDocs.totalHits)
termQuery = TermQuery(Term("contents","foobar"))
booleanQuery = BooleanQuery()
booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
- hits = searcher.search(booleanQuery)
- self.assertEqual(1, hits.length())
+ topDocs = searcher.search(booleanQuery, 50)
+ self.assertEqual(1, topDocs.totalHits)
searcher.close()
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
doc = Document()
doc.add(Field("contents", "map entry woo",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
writer.addDocument(doc)
doc = Document()
doc.add(Field("contents", "woo map entry",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
writer.addDocument(doc)
doc = Document()
doc.add(Field("contents", "map foobarword entry woo",
- Field.Store.YES, Field.Index.TOKENIZED,
+ Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.YES))
writer.addDocument(doc)
writer.optimize()
writer.close()
- searcher = IndexSearcher(directory)
+ searcher = IndexSearcher(directory, True)
termQuery = TermQuery(Term("contents", "woo"))
phraseQuery = PhraseQuery()
phraseQuery.add(Term("contents", "map"))
phraseQuery.add(Term("contents", "entry"))
- hits = searcher.search(termQuery)
- self.assertEqual(3, hits.length())
- hits = searcher.search(phraseQuery)
- self.assertEqual(2, hits.length())
+ topDocs = searcher.search(termQuery, 50)
+ self.assertEqual(3, topDocs.totalHits)
+ topDocs = searcher.search(phraseQuery, 50)
+ self.assertEqual(2, topDocs.totalHits)
booleanQuery = BooleanQuery()
booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
- hits = searcher.search(booleanQuery)
- self.assertEqual(2, hits.length())
+ topDocs = searcher.search(booleanQuery, 50)
+ self.assertEqual(2, topDocs.totalHits)
booleanQuery = BooleanQuery()
booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
- hits = searcher.search(booleanQuery)
- self.assertEqual(2, hits.length())
+ topDocs = searcher.search(booleanQuery, 50)
+ self.assertEqual(2, topDocs.totalHits)
searcher.close()
directory.close()
@@ -257,7 +261,7 @@
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_PositionIncrement.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PositionIncrement.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PositionIncrement.py (original)
+++ lucene/pylucene/trunk/test/test_PositionIncrement.py Tue Oct 13 07:38:10 2009
@@ -24,20 +24,26 @@
def testSetPosition(self):
class _analyzer(PythonAnalyzer):
- def tokenStream(self, fieldName, reader):
+ def tokenStream(_self, fieldName, reader):
class _tokenStream(PythonTokenStream):
- def __init__(self):
- super(_tokenStream, self).__init__()
- self.TOKENS = ["1", "2", "3", "4", "5"]
- self.INCREMENTS = [1, 2, 1, 0, 1]
- self.i = 0
- def next(self):
- if self.i == len(self.TOKENS):
- return None
- t = Token(self.TOKENS[self.i], self.i, self.i)
- t.setPositionIncrement(self.INCREMENTS[self.i])
- self.i += 1
- return t
+ def __init__(self_):
+ super(_tokenStream, self_).__init__()
+ self_.TOKENS = ["1", "2", "3", "4", "5"]
+ self_.INCREMENTS = [1, 2, 1, 0, 1]
+ self_.i = 0
+ self_.posIncrAtt = self_.addAttribute(PositionIncrementAttribute.class_)
+ self_.termAtt = self_.addAttribute(TermAttribute.class_)
+ self_.offsetAtt = self_.addAttribute(OffsetAttribute.class_)
+ def incrementToken(self_):
+ if self_.i == len(self_.TOKENS):
+ return False
+ self_.termAtt.setTermBuffer(self_.TOKENS[self_.i])
+ self_.offsetAtt.setOffset(self_.i, self_.i)
+ self_.posIncrAtt.setPositionIncrement(self_.INCREMENTS[self_.i])
+ self_.i += 1
+ return True
+ def end(self):
+ pass
def reset(self):
pass
def close(self):
@@ -47,76 +53,286 @@
analyzer = _analyzer()
store = RAMDirectory()
- writer = IndexWriter(store, analyzer, True)
+ writer = IndexWriter(store, analyzer, True,
+ IndexWriter.MaxFieldLength.LIMITED)
d = Document()
d.add(Field("field", "bogus",
- Field.Store.YES, Field.Index.TOKENIZED))
+ Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(d)
writer.optimize()
writer.close()
- searcher = IndexSearcher(store)
+ searcher = IndexSearcher(store, True)
+ pos = searcher.getIndexReader().termPositions(Term("field", "1"))
+ pos.next()
+ # first token should be at position 0
+ self.assertEqual(0, pos.nextPosition())
+
+ pos = searcher.getIndexReader().termPositions(Term("field", "2"))
+ pos.next()
+ # second token should be at position 2
+ self.assertEqual(2, pos.nextPosition())
+
q = PhraseQuery()
q.add(Term("field", "1"))
q.add(Term("field", "2"))
- hits = searcher.search(q)
- self.assertEqual(0, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # same as previous, just specify positions explicitely.
+ q = PhraseQuery()
+ q.add(Term("field", "1"), 0)
+ q.add(Term("field", "2"), 1)
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # specifying correct positions should find the phrase.
+ q = PhraseQuery()
+ q.add(Term("field", "1"), 0)
+ q.add(Term("field", "2"), 2)
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "3"))
- hits = searcher.search(q)
- self.assertEqual(1, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "4"))
- hits = searcher.search(q)
- self.assertEqual(0, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # phrase query would find it when correct positions are specified.
+ q = PhraseQuery()
+ q.add(Term("field", "3"), 0)
+ q.add(Term("field", "4"), 0)
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
+
+ # phrase query should fail for non existing searched term
+ # even if there exist another searched terms in the same searched
+ # position.
+ q = PhraseQuery()
+ q.add(Term("field", "3"), 0)
+ q.add(Term("field", "9"), 0)
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # multi-phrase query should succed for non existing searched term
+ # because there exist another searched terms in the same searched
+ # position.
+
+ mq = MultiPhraseQuery()
+ mq.add([Term("field", "3"), Term("field", "9")], 0)
+ hits = searcher.search(mq, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "4"))
- hits = searcher.search(q)
- self.assertEqual(1, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "3"))
q.add(Term("field", "5"))
- hits = searcher.search(q)
- self.assertEqual(1, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "4"))
q.add(Term("field", "5"))
- hits = searcher.search(q)
- self.assertEqual(1, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
q = PhraseQuery()
q.add(Term("field", "2"))
q.add(Term("field", "5"))
- hits = searcher.search(q)
- self.assertEqual(0, hits.length())
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # should not find "1 2" because there is a gap of 1 in the index
+ qp = QueryParser("field", StopWhitespaceAnalyzer(False))
+ q = PhraseQuery.cast_(qp.parse("\"1 2\""))
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # omitted stop word cannot help because stop filter swallows the
+ # increments.
+ q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # query parser alone won't help, because stop filter swallows the
+ # increments.
+ qp.setEnablePositionIncrements(True)
+ q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # stop filter alone won't help, because query parser swallows the
+ # increments.
+ qp.setEnablePositionIncrements(False)
+ q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(0, len(hits))
+
+ # when both qp qnd stopFilter propagate increments, we should find
+ # the doc.
+ qp = QueryParser("field", StopWhitespaceAnalyzer(True))
+ qp.setEnablePositionIncrements(True)
+ q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+ hits = searcher.search(q, None, 1000).scoreDocs
+ self.assertEqual(1, len(hits))
+
+ def testPayloadsPos0(self):
+
+ for x in xrange(2):
+ dir = RAMDirectory()
+ writer = IndexWriter(dir, TestPayloadAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
+ if x == 1:
+ writer.setAllowMinus1Position()
+
+ doc = Document()
+ doc.add(Field("content",
+ StringReader("a a b c d e a f g h i j a b k k")))
+ writer.addDocument(doc)
+
+ r = writer.getReader()
+
+ tp = r.termPositions(Term("content", "a"))
+ count = 0
+ self.assert_(tp.next())
+ # "a" occurs 4 times
+ self.assertEqual(4, tp.freq())
+
+ if x == 1:
+ expected = Integer.MAX_VALUE
+ else:
+ expected = 0
+
+ self.assertEqual(expected, tp.nextPosition())
+ if x == 1:
+ continue
+
+ self.assertEqual(1, tp.nextPosition())
+ self.assertEqual(3, tp.nextPosition())
+ self.assertEqual(6, tp.nextPosition())
+
+ # only one doc has "a"
+ self.assert_(not tp.next())
+
+ searcher = IndexSearcher(r)
+
+ stq1 = SpanTermQuery(Term("content", "a"))
+ stq2 = SpanTermQuery(Term("content", "k"))
+ sqs = [stq1, stq2]
+ snq = SpanNearQuery(sqs, 30, False)
+
+ count = 0
+ sawZero = False
+
+ pspans = snq.getSpans(searcher.getIndexReader())
+ while pspans.next():
+ payloads = pspans.getPayload()
+ sawZero |= pspans.start() == 0
+
+ it = payloads.iterator()
+ while it.hasNext():
+ count += 1
+ it.next()
+
+ self.assertEqual(5, count)
+ self.assert_(sawZero)
+
+ spans = snq.getSpans(searcher.getIndexReader())
+ count = 0
+ sawZero = False
+ while spans.next():
+ count += 1
+ sawZero |= spans.start() == 0
+
+ self.assertEqual(4, count)
+ self.assert_(sawZero)
+
+ sawZero = False
+ psu = PayloadSpanUtil(searcher.getIndexReader())
+ pls = psu.getPayloadsForQuery(snq)
+ count = pls.size()
+ it = pls.iterator()
+ while it.hasNext():
+ s = str(String(JArray('byte').cast_(it.next())))
+ sawZero |= s == "pos: 0"
+
+ self.assertEqual(5, count)
+ self.assert_(sawZero)
+ writer.close()
+ searcher.getIndexReader().close()
+ dir.close()
+
+
+class StopWhitespaceAnalyzer(PythonAnalyzer):
+
+ def __init__(self, enablePositionIncrements):
+ super(StopWhitespaceAnalyzer, self).__init__()
+
+ self.enablePositionIncrements = enablePositionIncrements
+ self.a = WhitespaceAnalyzer()
+
+ def tokenStream(self, fieldName, reader):
+
+ ts = self.a.tokenStream(fieldName, reader)
+ return StopFilter(self.enablePositionIncrements, ts, ["stop"])
+
+
+class TestPayloadAnalyzer(PythonAnalyzer):
+
+ def tokenStream(self, fieldName, reader):
+
+ result = LowerCaseTokenizer(reader)
+ return PayloadFilter(result, fieldName)
+
+
+class PayloadFilter(PythonTokenFilter):
+
+ def __init__(self, input, fieldName):
+ super(PayloadFilter, self).__init__(input)
+ self.input = input
+
+ self.fieldName = fieldName
+ self.pos = 0
+ self.i = 0
+ self.posIncrAttr = input.addAttribute(PositionIncrementAttribute.class_)
+ self.payloadAttr = input.addAttribute(PayloadAttribute.class_)
+ self.termAttr = input.addAttribute(TermAttribute.class_)
+
+ def incrementToken(self):
+
+ if self.input.incrementToken():
+ bytes = JArray('byte')("pos: %d" %(self.pos))
+ self.payloadAttr.setPayload(Payload(bytes))
+
+ if self.i % 2 == 1:
+ posIncr = 1
+ else:
+ posIncr = 0
+
+ self.posIncrAttr.setPositionIncrement(posIncr)
+ self.pos += posIncr
+ self.i += 1
+ return True
- def testIncrementingPositions(self):
- """
- Basic analyzer behavior should be to keep sequential terms in one
- increment from one another.
- """
-
- analyzer = WhitespaceAnalyzer()
- ts = analyzer.tokenStream("field",
- StringReader(u"one two three four five"))
-
- for token in ts:
- self.assertEqual(1, token.getPositionIncrement(),
- token.termText())
+ return False
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_PrefixFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PrefixFilter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PrefixFilter.py (original)
+++ lucene/pylucene/trunk/test/test_PrefixFilter.py Tue Oct 13 07:38:10 2009
@@ -30,12 +30,13 @@
"/Computers/Mac/Two",
"/Computers/Windows"]
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
for category in categories:
doc = Document()
doc.add(Field("category", category,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
writer.close()
@@ -43,63 +44,63 @@
# PrefixFilter combined with ConstantScoreQuery
filter = PrefixFilter(Term("category", "/Computers"))
query = ConstantScoreQuery(filter)
- searcher = IndexSearcher(directory)
- hits = searcher.search(query)
- self.assertEqual(4, hits.length(),
+ searcher = IndexSearcher(directory, True)
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(4, topDocs.totalHits,
"All documents in /Computers category and below")
# test middle of values
filter = PrefixFilter(Term("category", "/Computers/Mac"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(2, hits.length(), "Two in /Computers/Mac")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(2, topDocs.totalHits, "Two in /Computers/Mac")
# test start of values
filter = PrefixFilter(Term("category", "/Computers/Linux"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(1, hits.length(), "One in /Computers/Linux")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits, "One in /Computers/Linux")
# test end of values
filter = PrefixFilter(Term("category", "/Computers/Windows"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(1, hits.length(), "One in /Computers/Windows")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits, "One in /Computers/Windows")
# test non-existant
filter = PrefixFilter(Term("category", "/Computers/ObsoleteOS"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(0, hits.length(), "no documents")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits, "no documents")
# test non-existant, before values
filter = PrefixFilter(Term("category", "/Computers/AAA"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(0, hits.length(), "no documents")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits, "no documents")
# test non-existant, after values
filter = PrefixFilter(Term("category", "/Computers/ZZZ"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(0, hits.length(), "no documents")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits, "no documents")
# test zero-length prefix
filter = PrefixFilter(Term("category", ""))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(4, hits.length(), "all documents")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(4, topDocs.totalHits, "all documents")
# test non-existant field
filter = PrefixFilter(Term("nonexistantfield", "/Computers"))
query = ConstantScoreQuery(filter)
- hits = searcher.search(query)
- self.assertEqual(0, hits.length(), "no documents")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(0, topDocs.totalHits, "no documents")
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True:
Modified: lucene/pylucene/trunk/test/test_PrefixQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PrefixQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PrefixQuery.py (original)
+++ lucene/pylucene/trunk/test/test_PrefixQuery.py Tue Oct 13 07:38:10 2009
@@ -26,29 +26,30 @@
directory = RAMDirectory()
categories = ["/Computers", "/Computers/Mac", "/Computers/Windows"]
- writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+ writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+ IndexWriter.MaxFieldLength.LIMITED)
for category in categories:
doc = Document()
doc.add(Field("category", category,
- Field.Store.YES, Field.Index.UN_TOKENIZED))
+ Field.Store.YES, Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
writer.close()
query = PrefixQuery(Term("category", "/Computers"))
- searcher = IndexSearcher(directory)
- hits = searcher.search(query)
- self.assertEqual(3, hits.length(),
+ searcher = IndexSearcher(directory, True)
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(3, topDocs.totalHits,
"All documents in /Computers category and below")
query = PrefixQuery(Term("category", "/Computers/Mac"))
- hits = searcher.search(query)
- self.assertEqual(1, hits.length(), "One in /Computers/Mac")
+ topDocs = searcher.search(query, 50)
+ self.assertEqual(1, topDocs.totalHits, "One in /Computers/Mac")
if __name__ == "__main__":
import sys, lucene
- lucene.initVM(lucene.CLASSPATH)
+ lucene.initVM()
if '-loop' in sys.argv:
sys.argv.remove('-loop')
while True: