You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2009/10/13 09:38:11 UTC

svn commit: r824626 [1/2] - in /lucene/pylucene/trunk: ./ java/org/apache/pylucene/analysis/ java/org/apache/pylucene/search/ java/org/apache/pylucene/store/ test/

Author: vajda
Date: Tue Oct 13 07:38:10 2009
New Revision: 824626

URL: http://svn.apache.org/viewvc?rev=824626&view=rev
Log:
 - unit tests ported to new API
 - removed InstantiatedIndex contrib from default build
 - with (partial) JCC Java generics support, a lot less downcasting is needed

Added:
    lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py   (with props)
    lucene/pylucene/trunk/test/test_BinaryDocument.py   (with props)
Removed:
    lucene/pylucene/trunk/test/test_Compress.py
Modified:
    lucene/pylucene/trunk/CHANGES
    lucene/pylucene/trunk/Makefile
    lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java
    lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java
    lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java
    lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
    lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
    lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
    lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java
    lucene/pylucene/trunk/test/test_Analyzers.py
    lucene/pylucene/trunk/test/test_Binary.py
    lucene/pylucene/trunk/test/test_BooleanOr.py
    lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py
    lucene/pylucene/trunk/test/test_BooleanQuery.py
    lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
    lucene/pylucene/trunk/test/test_DocBoost.py
    lucene/pylucene/trunk/test/test_FilteredQuery.py
    lucene/pylucene/trunk/test/test_FuzzyQuery.py
    lucene/pylucene/trunk/test/test_Highlighter.py
    lucene/pylucene/trunk/test/test_Not.py
    lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py
    lucene/pylucene/trunk/test/test_PhraseQuery.py
    lucene/pylucene/trunk/test/test_PositionIncrement.py
    lucene/pylucene/trunk/test/test_PrefixFilter.py
    lucene/pylucene/trunk/test/test_PrefixQuery.py
    lucene/pylucene/trunk/test/test_PyLucene.py
    lucene/pylucene/trunk/test/test_PyLuceneThread.py
    lucene/pylucene/trunk/test/test_PythonDirectory.py
    lucene/pylucene/trunk/test/test_PythonQueryParser.py
    lucene/pylucene/trunk/test/test_RangeFilter.py
    lucene/pylucene/trunk/test/test_RangeQuery.py
    lucene/pylucene/trunk/test/test_RegexQuery.py
    lucene/pylucene/trunk/test/test_RewriteQuery.py
    lucene/pylucene/trunk/test/test_RuntimeException.py
    lucene/pylucene/trunk/test/test_Similarity.py
    lucene/pylucene/trunk/test/test_Sort.py
    lucene/pylucene/trunk/test/test_StopAnalyzer.py
    lucene/pylucene/trunk/test/test_StopWords.py
    lucene/pylucene/trunk/test/test_ThaiAnalyzer.py
    lucene/pylucene/trunk/test/test_bug1564.py
    lucene/pylucene/trunk/test/test_bug1763.py
    lucene/pylucene/trunk/test/test_bug1842.py

Modified: lucene/pylucene/trunk/CHANGES
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/CHANGES?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/CHANGES (original)
+++ lucene/pylucene/trunk/CHANGES Tue Oct 13 07:38:10 2009
@@ -1,4 +1,11 @@
 
+Version 2.9.0 -> 
+----------------------
+ - unit tests ported to new API
+ - removed InstantiatedIndex contrib from default build
+ - with (partial) JCC Java generics support, a lot less downcasting is needed
+ - 
+
 Version 2.4.1 -> 2.9.0
 ----------------------
  - renamed the Highlighter's SpanScorer class to HighlighterSpanScorer

Modified: lucene/pylucene/trunk/Makefile
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/Makefile?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/Makefile (original)
+++ lucene/pylucene/trunk/Makefile Tue Oct 13 07:38:10 2009
@@ -14,10 +14,10 @@
 # site-packages directory.
 #
 
-VERSION=2.9.0-1
+VERSION=3.0.0-1
 LUCENE_SVN_VER=HEAD
-LUCENE_VER=2.9.0
-LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/java/tags/lucene_2_9_0
+LUCENE_VER=3.0.0
+LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/java/trunk
 PYLUCENE:=$(shell pwd)
 LUCENE=lucene-java-$(LUCENE_VER)
 
@@ -48,6 +48,13 @@
 #JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64
 #NUM_FILES=2
 
+# Mac OS X 10.6 (MacPorts 1.8.0 64-bit Python 2.6, Java 1.6)
+#PREFIX_PYTHON=/opt/local
+#ANT=ant
+#PYTHON=$(PREFIX_PYTHON)/bin/python
+#JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64
+#NUM_FILES=2
+
 # Mac OS X 10.6 (64-bit and 32-bit Python 2.6 together, Java 1.6)
 #PREFIX_PYTHON=/usr
 #ANT=ant
@@ -128,7 +135,6 @@
 MEMORY_JAR=$(LUCENE)/build/contrib/memory/lucene-memory-$(LUCENE_VER).jar
 REGEX_JAR=$(LUCENE)/build/contrib/regex/lucene-regex-$(LUCENE_VER).jar
 QUERIES_JAR=$(LUCENE)/build/contrib/queries/lucene-queries-$(LUCENE_VER).jar
-INSTANTIATED_JAR=$(LUCENE)/build/contrib/instantiated/lucene-instantiated-$(LUCENE_VER).jar
 EXTENSIONS_JAR=build/jar/extensions.jar
 
 
@@ -177,15 +183,12 @@
 $(QUERIES_JAR): $(LUCENE_JAR)
 	cd $(LUCENE)/contrib/queries; $(ANT) -Dversion=$(LUCENE_VER)
 
-$(INSTANTIATED_JAR): $(LUCENE_JAR)
-	cd $(LUCENE)/contrib/instantiated; $(ANT) -Dversion=$(LUCENE_VER)
-
 $(EXTENSIONS_JAR): $(LUCENE_JAR)
 	$(ANT) -f extensions.xml -Dlucene.dir=$(LUCENE)
 
 JARS=$(LUCENE_JAR) $(SNOWBALL_JAR) $(ANALYZERS_JAR) \
      $(HIGHLIGHTER_JAR) $(MEMORY_JAR) \
-     $(REGEX_JAR) $(QUERIES_JAR) $(INSTANTIATED_JAR) $(EXTENSIONS_JAR)
+     $(REGEX_JAR) $(QUERIES_JAR) $(EXTENSIONS_JAR)
 
 
 jars: $(JARS)

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenFilter.java Tue Oct 13 07:38:10 2009
@@ -46,6 +46,6 @@
     }
 
     public native void pythonDecRef();
-    public native Token next()
+    public native boolean incrementToken()
         throws IOException;
 }

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenStream.java Tue Oct 13 07:38:10 2009
@@ -44,7 +44,13 @@
     }
 
     public native void pythonDecRef();
-    public native Token next();
-    public native void reset();
-    public native void close();
+
+    public native boolean incrementToken()
+        throws IOException;
+    public native void end()
+        throws IOException;
+    public native void reset()
+        throws IOException;
+    public native void close()
+        throws IOException;
 }

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonTokenizer.java Tue Oct 13 07:38:10 2009
@@ -47,6 +47,6 @@
 
     public native void pythonDecRef();
 
-    public native Token next()
+    public native boolean incrementToken()
         throws IOException;
 }

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java Tue Oct 13 07:38:10 2009
@@ -15,13 +15,12 @@
 
 package org.apache.pylucene.search;
 
+import java.io.IOException;
+
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.index.IndexReader;
-import java.util.BitSet;
+import org.apache.lucene.util.DocIdBitSet;
 
-/**
- * @author Andi Vajda
- */
 
 public class PythonFilter extends Filter {
 
@@ -47,5 +46,6 @@
     }
 
     public native void pythonDecRef();
-    public native BitSet bits(IndexReader reader);
+    public native DocIdBitSet getDocIdSet(IndexReader reader)
+        throws IOException;
 }

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java Tue Oct 13 07:38:10 2009
@@ -15,6 +15,8 @@
 
 package org.apache.pylucene.store;
 
+import java.io.IOException;
+
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -45,20 +47,27 @@
     }
 
     public native void pythonDecRef();
-    public native void close();
-    public native IndexOutput createOutput(String name);
-    public native void deleteFile(String name);
-    public native boolean fileExists(String name);
-    public native long fileLength(String name);
-    public native long fileModified(String name);
-    public native String[] list();
-    public native Lock makeLock(String name);
-    public native IndexInput openInput(String name);
-    public native IndexInput openInput(String name, int bufferSize);
-    public native void touchFile(String name);
 
-    /* no longer implemented, deprecated in Lucene 2.1 */
-    public void renameFile(String from, String to)
-    {
-    }
+    public native void close()
+        throws IOException;
+    public native IndexOutput createOutput(String name)
+        throws IOException;
+    public native void deleteFile(String name)
+        throws IOException;
+    public native boolean fileExists(String name)
+        throws IOException;
+    public native long fileLength(String name)
+        throws IOException;
+    public native long fileModified(String name)
+        throws IOException;
+    public native String[] listAll()
+        throws IOException;
+    public native IndexInput openInput(String name)
+        throws IOException;
+    public native IndexInput openInput(String name, int bufferSize)
+        throws IOException;
+    public native void touchFile(String name)
+        throws IOException;
+    public native void sync(String name) 
+        throws IOException;
 }

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java Tue Oct 13 07:38:10 2009
@@ -15,8 +15,10 @@
 
 package org.apache.pylucene.store;
 
+import java.io.IOException;
 import org.apache.lucene.store.BufferedIndexInput;
 
+
 public class PythonIndexInput extends BufferedIndexInput {
 
     private long pythonObject;
@@ -40,14 +42,19 @@
         pythonDecRef();
     }
 
-    public native Object clone();
-    public native void close();
     public native void pythonDecRef();
+
+    public native Object clone();
     public native long length();
-    public native byte[] readInternal(int length, long pos);
-    public native void seekInternal(long pos);
+    public native void close()
+        throws IOException;
+    public native byte[] readInternal(int length, long pos)
+        throws IOException;
+    public native void seekInternal(long pos)
+        throws IOException;
 
     protected void readInternal(byte[] b, int offset, int length)
+        throws IOException
     {
         byte[] data = readInternal(length, getFilePointer());
         System.arraycopy(data, 0, b, offset, data.length);

Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexOutput.java Tue Oct 13 07:38:10 2009
@@ -50,12 +50,17 @@
     }
 
     public native void pythonDecRef();
-    public native long length();
-    public native void flushBuffer(byte[] data);
-    public native void seekInternal(long pos);
-    public native void close();
+    public native long length()
+        throws IOException;
+    public native void flushBuffer(byte[] data)
+        throws IOException;
+    public native void seekInternal(long pos)
+        throws IOException;
+    public native void close()
+        throws IOException;
 
     protected void flushBuffer(byte[] b, int offset, int len)
+        throws IOException
     {
         byte[] data = new byte[len];
         System.arraycopy(b, offset, data, 0, len);

Added: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py?rev=824626&view=auto
==============================================================================
--- lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py (added)
+++ lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py Tue Oct 13 07:38:10 2009
@@ -0,0 +1,104 @@
+# ====================================================================
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+# ====================================================================
+
+from unittest import TestCase, main
+from lucene import *
+
+class BaseTokenStreamTestCase(TestCase):
+    """
+    Base class for all Lucene unit tests that use TokenStreams.  
+    """
+
+    def _assertTokenStreamContents(self, ts, output,
+                                   startOffsets=None, endOffsets=None,
+                                   types=None, posIncrements=None):
+
+        self.assert_(output is not None)
+        self.assert_(ts.hasAttribute(TermAttribute.class_),
+                                     "has TermAttribute")
+
+        termAtt = ts.getAttribute(TermAttribute.class_)
+
+        offsetAtt = None
+        if startOffsets is not None or endOffsets is not None:
+            self.assert_(ts.hasAttribute(OffsetAttribute.class_),
+                                         "has OffsetAttribute")
+            offsetAtt = ts.getAttribute(OffsetAttribute.class_)
+    
+        typeAtt = None
+        if types is not None:
+            self.assert_(ts.hasAttribute(TypeAttribute.class_),
+                         "has TypeAttribute")
+            typeAtt = ts.getAttribute(TypeAttribute.class_)
+    
+        posIncrAtt = None
+        if posIncrements is not None:
+            self.assert_(ts.hasAttribute(PositionIncrementAttribute.class_),
+                         "has PositionIncrementAttribute")
+            posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class_)
+    
+        ts.reset()
+        for i in xrange(len(output)):
+            # extra safety to enforce, that the state is not preserved and
+            # also assign bogus values
+            ts.clearAttributes()
+            termAtt.setTermBuffer("bogusTerm")
+            if offsetAtt is not None:
+                offsetAtt.setOffset(14584724, 24683243)
+            if typeAtt is not None:
+                typeAtt.setType("bogusType")
+            if posIncrAtt is not None:
+                posIncrAtt.setPositionIncrement(45987657)
+      
+            self.assert_(ts.incrementToken(), "token %d exists" %(i))
+            self.assertEqual(output[i], termAtt.term(), "term %d" %(i))
+            if startOffsets is not None:
+                self.assertEqual(startOffsets[i], offsetAtt.startOffset(),
+                                 "startOffset %d" %(i))
+            if endOffsets is not None:
+                self.assertEqual(endOffsets[i], offsetAtt.endOffset(),
+                                 "endOffset %d" %(i))
+            if types is not None:
+                self.assertEqual(types[i], typeAtt.type(), "type %d" %(i))
+            if posIncrements is not None:
+                self.assertEqual(posIncrements[i],
+                                 posIncrAtt.getPositionIncrement(),
+                                 "posIncrement %d" %(i))
+
+        self.assert_(not ts.incrementToken(), "end of stream")
+        ts.end()
+        ts.close()
+
+    def _assertAnalyzesTo(self, a, input, output,
+                          startOffsets=None, endOffsets=None,
+                          types=None, posIncrements=None):
+
+        ts = a.tokenStream("dummy", StringReader(input))
+        self._assertTokenStreamContents(ts, output, startOffsets, endOffsets,
+                                        types, posIncrements)
+
+    def _assertAnalyzesToReuse(self, a, input, output,
+                               startOffsets=None, endOffsets=None,
+                               types=None, posIncrements=None):
+
+        ts = a.reusableTokenStream("dummy", StringReader(input))
+        self._assertTokenStreamContents(ts, output, startOffsets, endOffsets,
+                                        types, posIncrements)
+  
+    # simple utility method for testing stemmers
+    def _checkOneTerm(self, a, input, expected):
+        self._assertAnalyzesTo(a, input, JArray('string')(expected))
+  
+    def _checkOneTermReuse(self, a, input, expected):
+        self._assertAnalyzesToReuse(a, input, JArray('string')(expected))

Propchange: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: lucene/pylucene/trunk/test/test_Analyzers.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Analyzers.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Analyzers.py (original)
+++ lucene/pylucene/trunk/test/test_Analyzers.py Tue Oct 13 07:38:10 2009
@@ -12,27 +12,16 @@
 #   limitations under the License.
 # ====================================================================
 
-from unittest import TestCase, main
+from unittest import main
+from BaseTokenStreamTestCase import BaseTokenStreamTestCase;
 from lucene import *
 
 
-class AnalyzersTestCase(TestCase):
+class AnalyzersTestCase(BaseTokenStreamTestCase):
     """
     Unit tests ported from Java Lucene
     """
 
-    def _assertAnalyzesTo(self, a, input, output):
-
-        ts = a.tokenStream("dummy", StringReader(input))
-        for string in output:
-            t = ts.next()
-            self.assert_(t is not None)
-            self.assertEqual(t.termText(), string)
-
-        self.assert_(not list(ts))
-        ts.close()
-
-
     def testSimple(self):
 
         a = SimpleAnalyzer()
@@ -53,6 +42,7 @@
         self._assertAnalyzesTo(a, "\"QUOTED\" word", 
                                [ "quoted", "word" ])
 
+
     def testNull(self):
 
         a = WhitespaceAnalyzer()
@@ -81,10 +71,53 @@
         self._assertAnalyzesTo(a, "foo a bar such FOO THESE BAR", 
                                [ "foo", "bar", "foo", "bar" ])
 
+    def _verifyPayload(self, ts):
+
+        payloadAtt = ts.getAttribute(PayloadAttribute.class_)
+        b = 0
+        while True:
+            b += 1
+            if not ts.incrementToken():
+                break
+            self.assertEqual(b, payloadAtt.getPayload().toByteArray()[0])
+
+    # Make sure old style next() calls result in a new copy of payloads
+    def testPayloadCopy(self):
+
+        s = "how now brown cow"
+        ts = WhitespaceTokenizer(StringReader(s))
+        ts = PayloadSetter(ts)
+        self._verifyPayload(ts)
+
+        ts = WhitespaceTokenizer(StringReader(s))
+        ts = PayloadSetter(ts)
+        self._verifyPayload(ts)
+
+
+class PayloadSetter(PythonTokenFilter):
+
+    def __init__(self, input):
+        super(PayloadSetter, self).__init__(input)
+
+        self.input = input
+        self.payloadAtt = self.addAttribute(PayloadAttribute.class_)
+        self.data = JArray('byte')(1)
+        self.p = Payload(self.data, 0, 1)
+
+    def incrementToken(self):
+
+        if not self.input.incrementToken():
+            return False
+
+        self.payloadAtt.setPayload(self.p)
+        self.data[0] += 1;
+
+        return True
+
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_Binary.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Binary.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Binary.py (original)
+++ lucene/pylucene/trunk/test/test_Binary.py Tue Oct 13 07:38:10 2009
@@ -21,22 +21,22 @@
 
     def binary(self, b):
 
-        c = JArray('byte')([chr(a) for a in b])
+        c = JArray('byte')(b)
         field = Field("bin", c, Field.Store.YES)
-        v = field.binaryValue()
-        assert c == v and b == [ord(a) for a in v]
+        v = field.binaryValue
+        assert c == v and b == [a for a in v]
 
     def testBinary(self):
 
         self.binary([66, 90, 104, 57, 49, 65, 89, 38,
-                     83, 89, 105, 56, 95, 75, 0, 0, 14, 215, 128])
+                     83, 89, 105, 56, 95, 75, 0, 0, 14, -41, -128])
         self.binary([])
         self.binary([0, 0, 0])
 
 
 if __name__ == '__main__':
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Added: lucene/pylucene/trunk/test/test_BinaryDocument.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BinaryDocument.py?rev=824626&view=auto
==============================================================================
--- lucene/pylucene/trunk/test/test_BinaryDocument.py (added)
+++ lucene/pylucene/trunk/test/test_BinaryDocument.py Tue Oct 13 07:38:10 2009
@@ -0,0 +1,120 @@
+# ====================================================================
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+# ====================================================================
+
+from unittest import TestCase, main
+from lucene import *
+
+
+class TestBinaryDocument(TestCase):
+
+    binaryValStored = "this text will be stored as a byte array in the index"
+    binaryValCompressed = "this text will be also stored and compressed as a byte array in the index"
+  
+    def testBinaryFieldInIndex(self):
+
+        bytes = JArray('byte')(self.binaryValStored)
+        binaryFldStored = Field("binaryStored", bytes, 
+                                Field.Store.YES)
+        stringFldStored = Field("stringStored", self.binaryValStored,
+                                Field.Store.YES, Field.Index.NO,
+                                Field.TermVector.NO)
+
+        try:
+            # binary fields with store off are not allowed
+            Field("fail", bytes, Field.Store.NO)
+            self.fail()
+        except JavaError, e:
+            self.assertEqual(e.getJavaException().getClass().getName(),
+                             'java.lang.IllegalArgumentException')
+    
+        doc = Document()
+        doc.add(binaryFldStored)
+        doc.add(stringFldStored)
+
+        # test for field count
+        self.assertEqual(2, doc.fields.size())
+    
+        # add the doc to a ram index
+        dir = RAMDirectory()
+        writer = IndexWriter(dir, StandardAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
+        writer.addDocument(doc)
+        writer.close()
+    
+        # open a reader and fetch the document
+        reader = IndexReader.open(dir, False)
+        docFromReader = reader.document(0)
+        self.assert_(docFromReader is not None)
+    
+        # fetch the binary stored field and compare it's content with the
+        # original one
+        binaryFldStoredTest = str(String(docFromReader.getBinaryValue("binaryStored")))
+        self.assertEqual(binaryFldStoredTest, self.binaryValStored)
+        
+        # fetch the string field and compare it's content with the original
+        # one
+        stringFldStoredTest = docFromReader.get("stringStored")
+        self.assertEqual(stringFldStoredTest, self.binaryValStored)
+    
+        # delete the document from index
+        reader.deleteDocument(0)
+        self.assertEqual(0, reader.numDocs())
+    
+        reader.close()
+        dir.close()
+  
+    def testCompressionTools(self):
+
+        bytes = JArray('byte')(self.binaryValCompressed)
+        binaryFldCompressed = Field("binaryCompressed", CompressionTools.compress(bytes), Field.Store.YES)
+        stringFldCompressed = Field("stringCompressed", CompressionTools.compressString(self.binaryValCompressed), Field.Store.YES)
+    
+        doc = Document()
+        doc.add(binaryFldCompressed)
+        doc.add(stringFldCompressed)
+    
+        # add the doc to a ram index
+        dir = RAMDirectory()
+        writer = IndexWriter(dir, StandardAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
+        writer.addDocument(doc)
+        writer.close()
+    
+        # open a reader and fetch the document
+        reader = IndexReader.open(dir, False)
+        docFromReader = reader.document(0)
+        self.assert_(docFromReader is not None)
+    
+        # fetch the binary compressed field and compare it's content with
+        # the original one
+        binaryFldCompressedTest = str(String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed"))))
+        self.assertEqual(binaryFldCompressedTest, self.binaryValCompressed)
+        self.assertEqual(CompressionTools.decompressString(docFromReader.getBinaryValue("stringCompressed")), self.binaryValCompressed)
+
+        reader.close()
+        dir.close()
+
+
+if __name__ == '__main__':
+    import sys, lucene
+    lucene.initVM()
+    if '-loop' in sys.argv:
+        sys.argv.remove('-loop')
+        while True:
+            try:
+                main()
+            except:
+                pass
+    else:
+        main()

Propchange: lucene/pylucene/trunk/test/test_BinaryDocument.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/pylucene/trunk/test/test_BinaryDocument.py
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: lucene/pylucene/trunk/test/test_BooleanOr.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanOr.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanOr.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanOr.py Tue Oct 13 07:38:10 2009
@@ -38,23 +38,24 @@
     def setUp(self):
 
         rd = RAMDirectory()
-        writer = IndexWriter(rd, StandardAnalyzer(), True)
+        writer = IndexWriter(rd, StandardAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
 
         d = Document()
         d.add(Field(self.FIELD_T,
                     "Optimize not deleting all files",
-                    Field.Store.YES, Field.Index.TOKENIZED))
+                    Field.Store.YES, Field.Index.ANALYZED))
         d.add(Field(self.FIELD_C,
                     "Deleted When I run an optimize in our production environment.",
-                    Field.Store.YES, Field.Index.TOKENIZED))
+                    Field.Store.YES, Field.Index.ANALYZED))
 
         writer.addDocument(d)
         writer.close()
 
-        self.searcher = IndexSearcher(rd)
+        self.searcher = IndexSearcher(rd, True)
 
     def search(self, q):
-        return self.searcher.search(q).length()
+        return self.searcher.search(q, 50).totalHits
 
     def testElements(self):
 
@@ -114,7 +115,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py Tue Oct 13 07:38:10 2009
@@ -52,7 +52,7 @@
 
             writer.close()
       
-            reader = IndexReader.open(directory)
+            reader = IndexReader.open(directory, True)
             query = PrefixQuery(Term("category", "foo"))
             rw1 = query.rewrite(reader)
       
@@ -69,7 +69,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_BooleanQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BooleanQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BooleanQuery.py (original)
+++ lucene/pylucene/trunk/test/test_BooleanQuery.py Tue Oct 13 07:38:10 2009
@@ -46,7 +46,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_CachingWrapperFilter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_CachingWrapperFilter.py (original)
+++ lucene/pylucene/trunk/test/test_CachingWrapperFilter.py Tue Oct 13 07:38:10 2009
@@ -24,18 +24,19 @@
     def testCachingWorks(self):
 
         dir = RAMDirectory()
-        writer = IndexWriter(dir, StandardAnalyzer(), True)
+        writer = IndexWriter(dir, StandardAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         writer.close()
 
-        reader = IndexReader.open(dir)
+        reader = IndexReader.open(dir, True)
 
         class mockFilter(PythonFilter):
             def __init__(self):
                 super(mockFilter, self).__init__()
                 self._wasCalled = False
-            def bits(self, reader):
+            def getDocIdSet(self, reader):
                 self._wasCalled = True;
-                return BitSet()
+                return DocIdBitSet(BitSet())
             def clear(self):
                 self._wasCalled = False
             def wasCalled(self):
@@ -45,12 +46,12 @@
         cacher = CachingWrapperFilter(filter)
 
         # first time, nested filter is called
-        cacher.bits(reader)
+        cacher.getDocIdSet(reader)
         self.assert_(filter.wasCalled(), "first time")
 
         # second time, nested filter should not be called
         filter.clear()
-        cacher.bits(reader)
+        cacher.getDocIdSet(reader)
         self.assert_(not filter.wasCalled(), "second time")
 
         reader.close()
@@ -58,7 +59,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_DocBoost.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_DocBoost.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_DocBoost.py (original)
+++ lucene/pylucene/trunk/test/test_DocBoost.py Tue Oct 13 07:38:10 2009
@@ -24,10 +24,11 @@
     def testDocBoost(self):
 
         store = RAMDirectory()
-        writer = IndexWriter(store, SimpleAnalyzer(), True)
+        writer = IndexWriter(store, SimpleAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
     
-        f1 = Field("field", "word", Field.Store.YES, Field.Index.TOKENIZED)
-        f2 = Field("field", "word", Field.Store.YES, Field.Index.TOKENIZED)
+        f1 = Field("field", "word", Field.Store.YES, Field.Index.ANALYZED)
+        f2 = Field("field", "word", Field.Store.YES, Field.Index.ANALYZED)
         f2.setBoost(2.0)
     
         d1 = Document()
@@ -55,8 +56,8 @@
             def collect(self, doc, score):
                 scores[doc] = score
 
-        IndexSearcher(store).search(TermQuery(Term("field", "word")),
-                                    hitCollector())
+        IndexSearcher(store, True).search(TermQuery(Term("field", "word")),
+                                          hitCollector())
     
         lastScore = 0.0
         for score in scores:
@@ -66,7 +67,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_FilteredQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_FilteredQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_FilteredQuery.py (original)
+++ lucene/pylucene/trunk/test/test_FilteredQuery.py Tue Oct 13 07:38:10 2009
@@ -24,52 +24,53 @@
     def setUp(self):
 
         self.directory = RAMDirectory()
-        writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
 
         doc = Document()
         doc.add(Field("field", "one two three four five",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         doc.add(Field("sorter", "b",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
                       
         writer.addDocument(doc)
 
         doc = Document()
         doc.add(Field("field", "one two three four",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         doc.add(Field("sorter", "d",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
 
         writer.addDocument(doc)
 
         doc = Document()
         doc.add(Field("field", "one two three y",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         doc.add(Field("sorter", "a",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
 
         writer.addDocument(doc)
 
         doc = Document()
         doc.add(Field("field", "one two x",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         doc.add(Field("sorter", "c",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
                       
         writer.addDocument(doc)
 
         writer.optimize()
         writer.close()
 
-        self.searcher = IndexSearcher(self.directory)
+        self.searcher = IndexSearcher(self.directory, True)
         self.query = TermQuery(Term("field", "three"))
 
         class filter(PythonFilter):
-            def bits(self, reader):
+            def getDocIdSet(self, reader):
                 bitset = BitSet(5)
                 bitset.set(1)
                 bitset.set(3)
-                return bitset
+                return DocIdBitSet(bitset)
 
         self.filter = filter()
 
@@ -81,44 +82,44 @@
     def testFilteredQuery(self):
 
         filteredquery = FilteredQuery(self.query, self.filter)
-        hits = self.searcher.search(filteredquery);
-        self.assertEqual(1, hits.length())
-        self.assertEqual(1, hits.id(0))
-
-        hits = self.searcher.search(filteredquery, Sort("sorter"))
-        self.assertEqual(1, hits.length())
-        self.assertEqual(1, hits.id(0))
+        topDocs = self.searcher.search(filteredquery, 50)
+        self.assertEqual(1, topDocs.totalHits)
+        self.assertEqual(1, topDocs.scoreDocs[0].doc)
+
+        topDocs = self.searcher.search(filteredquery, None, 50, Sort("sorter"))
+        self.assertEqual(1, topDocs.totalHits)
+        self.assertEqual(1, topDocs.scoreDocs[0].doc)
 
         filteredquery = FilteredQuery(TermQuery(Term("field", "one")),
                                       self.filter)
-        hits = self.searcher.search(filteredquery)
-        self.assertEqual(2, hits.length())
+        topDocs = self.searcher.search(filteredquery, 50)
+        self.assertEqual(2, topDocs.totalHits)
 
         filteredquery = FilteredQuery(TermQuery(Term("field", "x")),
                                       self.filter)
-        hits = self.searcher.search(filteredquery)
-        self.assertEqual(1, hits.length())
-        self.assertEqual(3, hits.id(0))
+        topDocs = self.searcher.search(filteredquery, 50)
+        self.assertEqual(1, topDocs.totalHits)
+        self.assertEqual(3, topDocs.scoreDocs[0].doc)
 
         filteredquery = FilteredQuery(TermQuery(Term("field", "y")),
                                       self.filter)
-        hits = self.searcher.search(filteredquery)
-        self.assertEqual(0, hits.length())
+        topDocs = self.searcher.search(filteredquery, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
     def testRangeQuery(self):
         """
         This tests FilteredQuery's rewrite correctness
         """
-        
-        rq = RangeQuery(Term("sorter", "b"), Term("sorter", "d"), True)
+
+        rq = TermRangeQuery("sorter", "b", "d", True, True)
         filteredquery = FilteredQuery(rq, self.filter)
-        hits = self.searcher.search(filteredquery)
-        self.assertEqual(2, hits.length())
+        scoreDocs = self.searcher.search(filteredquery, 1000).scoreDocs
+        self.assertEqual(2, len(scoreDocs))
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_FuzzyQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_FuzzyQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_FuzzyQuery.py (original)
+++ lucene/pylucene/trunk/test/test_FuzzyQuery.py Tue Oct 13 07:38:10 2009
@@ -22,15 +22,17 @@
     """
 
     def _addDoc(self, text, writer):
+
         doc = Document()
         doc.add(Field("field", text,
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         writer.addDocument(doc)
 
     def testDefaultFuzziness(self):
 
         directory = RAMDirectory()
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         self._addDoc("aaaaa", writer)
         self._addDoc("aaaab", writer)
         self._addDoc("aaabb", writer)
@@ -41,47 +43,47 @@
         writer.optimize()
         writer.close()
 
-        searcher = IndexSearcher(directory)
+        searcher = IndexSearcher(directory, True)
 
         query = FuzzyQuery(Term("field", "aaaaa"))
-        hits = searcher.search(query)
-        self.assertEqual(3, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(3, topDocs.totalHits)
 
         # not similar enough:
         query = FuzzyQuery(Term("field", "xxxxx"))
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
         # edit distance to "aaaaa" = 3
         query = FuzzyQuery(Term("field", "aaccc"))
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
         # query identical to a word in the index:
         query = FuzzyQuery(Term("field", "aaaaa"))
-        hits = searcher.search(query)
-        self.assertEqual(3, hits.length())
-        self.assertEqual(hits.doc(0).get("field"), "aaaaa")
+        scoreDocs = searcher.search(query, 50).scoreDocs
+        self.assertEqual(3, len(scoreDocs))
+        self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaa")
         # default allows for up to two edits:
-        self.assertEqual(hits.doc(1).get("field"), "aaaab")
-        self.assertEqual(hits.doc(2).get("field"), "aaabb")
+        self.assertEqual(searcher.doc(scoreDocs[1].doc).get("field"), "aaaab")
+        self.assertEqual(searcher.doc(scoreDocs[2].doc).get("field"), "aaabb")
 
         # query similar to a word in the index:
         query = FuzzyQuery(Term("field", "aaaac"))
-        hits = searcher.search(query)
-        self.assertEqual(3, hits.length())
-        self.assertEqual(hits.doc(0).get("field"), "aaaaa")
-        self.assertEqual(hits.doc(1).get("field"), "aaaab")
-        self.assertEqual(hits.doc(2).get("field"), "aaabb")
+        scoreDocs = searcher.search(query, 50).scoreDocs
+        self.assertEqual(3, len(scoreDocs))
+        self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaa")
+        self.assertEqual(searcher.doc(scoreDocs[1].doc).get("field"), "aaaab")
+        self.assertEqual(searcher.doc(scoreDocs[2].doc).get("field"), "aaabb")
 
         query = FuzzyQuery(Term("field", "ddddX"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
-        self.assertEqual(hits.doc(0).get("field"), "ddddd")
+        scoreDocs = searcher.search(query, 50).scoreDocs
+        self.assertEqual(1, len(scoreDocs))
+        self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "ddddd")
 
         # different field = no match:
         query = FuzzyQuery(Term("anotherfield", "ddddX"))
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
         searcher.close()
         directory.close()
@@ -89,37 +91,38 @@
     def testDefaultFuzzinessLong(self):
 
         directory = RAMDirectory()
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         self._addDoc("aaaaaaa", writer)
         self._addDoc("segment", writer)
         writer.optimize()
         writer.close()
-        searcher = IndexSearcher(directory)
+        searcher = IndexSearcher(directory, True)
 
         # not similar enough:
         query = FuzzyQuery(Term("field", "xxxxx"))
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
         # edit distance to "aaaaaaa" = 3, this matches because
         # the string is longer than
         # in testDefaultFuzziness so a bigger difference is allowed:
         query = FuzzyQuery(Term("field", "aaaaccc"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
-        self.assertEqual(hits.doc(0).get("field"), "aaaaaaa")
+        scoreDocs = searcher.search(query, 50).scoreDocs
+        self.assertEqual(1, len(scoreDocs))
+        self.assertEqual(searcher.doc(scoreDocs[0].doc).get("field"), "aaaaaaa")
 
         # no match, more than half of the characters is wrong:
         query = FuzzyQuery(Term("field", "aaacccc"))
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
         # "student" and "stellent" are indeed similar to "segment" by default:
         query = FuzzyQuery(Term("field", "student"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits)
         query = FuzzyQuery(Term("field", "stellent"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits)
 
         searcher.close()
         directory.close()
@@ -127,7 +130,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_Highlighter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Highlighter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Highlighter.py (original)
+++ lucene/pylucene/trunk/test/test_Highlighter.py Tue Oct 13 07:38:10 2009
@@ -56,8 +56,8 @@
         highlighter.setTextFragmenter(SimpleFragmenter(40))
         maxNumFragmentsRequired = 2
 
-        for i in range(0, self.hits.length()):
-            text = self.hits.doc(i).get(self.FIELD_NAME)
+        for scoreDoc in self.scoreDocs:
+            text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
             tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
                                                     StringReader(text))
 
@@ -79,14 +79,14 @@
         
     def doSearching(self, queryString):
 
-        searcher = IndexSearcher(self.ramDir)
+        self.searcher = IndexSearcher(self.ramDir, True)
         self.query = self.parser.parse(queryString)
         # for any multi-term queries to work (prefix, wildcard, range,
         # fuzzy etc) you must use a rewritten query!
         self.query = self.query.rewrite(self.reader)
 
         print "Searching for:", self.query.toString(self.FIELD_NAME)
-        self.hits = searcher.search(self.query)
+        self.scoreDocs = self.searcher.search(self.query, 100).scoreDocs
         self.numHighlights = 0
 
     def doStandardHighlights(self):
@@ -95,8 +95,8 @@
         
         highlighter = Highlighter(formatter, QueryScorer(self.query))
         highlighter.setTextFragmenter(SimpleFragmenter(20))
-        for i in range(0, self.hits.length()):
-            text = self.hits.doc(i).get(self.FIELD_NAME)
+        for scoreDoc in self.scoreDocs:
+            text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
             maxNumFragmentsRequired = 2
             fragmentSeparator = "..."
             tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
@@ -114,22 +114,23 @@
         
     def setUp(self):
 
-        self.analyzer=StandardAnalyzer()
+        self.analyzer = StandardAnalyzer()
         self.ramDir = RAMDirectory()
-        writer = IndexWriter(self.ramDir, self.analyzer, True)
+        writer = IndexWriter(self.ramDir, self.analyzer, True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         for text in self.texts:
             self.addDoc(writer, text)
 
         writer.optimize()
         writer.close()
-        self.reader = IndexReader.open(self.ramDir)
+        self.reader = IndexReader.open(self.ramDir, True)
         self.numHighlights = 0;
 
     def addDoc(self, writer, text):
 
         d = Document()
         f = Field(self.FIELD_NAME, text,
-                  Field.Store.YES, Field.Index.TOKENIZED,
+                  Field.Store.YES, Field.Index.ANALYZED,
                   Field.TermVector.YES)
         d.add(f)
         writer.addDocument(d)
@@ -137,7 +138,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_Not.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Not.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Not.py (original)
+++ lucene/pylucene/trunk/test/test_Not.py Tue Oct 13 07:38:10 2009
@@ -24,25 +24,26 @@
     def testNot(self):
 
         store = RAMDirectory()
-        writer = IndexWriter(store, SimpleAnalyzer(), True)
+        writer = IndexWriter(store, SimpleAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
 
         d1 = Document()
-        d1.add(Field("field", "a b", Field.Store.YES, Field.Index.TOKENIZED))
+        d1.add(Field("field", "a b", Field.Store.YES, Field.Index.ANALYZED))
 
         writer.addDocument(d1)
         writer.optimize()
         writer.close()
 
-        searcher = IndexSearcher(store)
+        searcher = IndexSearcher(store, True)
         query = QueryParser("field", SimpleAnalyzer()).parse("a NOT b")
 
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py (original)
+++ lucene/pylucene/trunk/test/test_PerFieldAnalyzerWrapper.py Tue Oct 13 07:38:10 2009
@@ -28,19 +28,22 @@
         analyzer.addAnalyzer("special", SimpleAnalyzer())
 
         tokenStream = analyzer.tokenStream("field", StringReader(text))
-        token = tokenStream.next()
-        self.assertEqual("Qwerty", token.termText(),
+        termAtt = tokenStream.getAttribute(TermAttribute.class_)
+
+        self.assert_(tokenStream.incrementToken())
+        self.assertEqual("Qwerty", termAtt.term(),
                          "WhitespaceAnalyzer does not lowercase")
 
         tokenStream = analyzer.tokenStream("special", StringReader(text))
-        token = tokenStream.next()
-        self.assertEqual("qwerty", token.termText(),
+        termAtt = tokenStream.getAttribute(TermAttribute.class_)
+        self.assert_(tokenStream.incrementToken())
+        self.assertEqual("qwerty", termAtt.term(),
                          "SimpleAnalyzer lowercases")
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_PhraseQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PhraseQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PhraseQuery.py (original)
+++ lucene/pylucene/trunk/test/test_PhraseQuery.py Tue Oct 13 07:38:10 2009
@@ -24,17 +24,18 @@
     def setUp(self):
 
         self.directory = RAMDirectory()
-        writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(self.directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
     
         doc = Document()
         doc.add(Field("field", "one two three four five",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         writer.addDocument(doc)
     
         writer.optimize()
         writer.close()
 
-        self.searcher = IndexSearcher(self.directory)
+        self.searcher = IndexSearcher(self.directory, True)
         self.query = PhraseQuery()
 
     def tearDown(self):
@@ -47,16 +48,16 @@
         self.query.setSlop(2)
         self.query.add(Term("field", "one"))
         self.query.add(Term("field", "five"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(0, hits.length())
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(0, topDocs.totalHits)
 
     def testBarelyCloseEnough(self):
 
         self.query.setSlop(3)
         self.query.add(Term("field", "one"))
         self.query.add(Term("field", "five"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length())
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits)
 
     def testExact(self):
         """
@@ -66,14 +67,14 @@
         # slop is zero by default
         self.query.add(Term("field", "four"))
         self.query.add(Term("field", "five"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length(), "exact match")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits, "exact match")
 
         self.query = PhraseQuery()
         self.query.add(Term("field", "two"))
         self.query.add(Term("field", "one"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(0, hits.length(), "reverse not exact")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(0, topDocs.totalHits, "reverse not exact")
 
     def testSlop1(self):
 
@@ -81,8 +82,8 @@
         self.query.setSlop(1)
         self.query.add(Term("field", "one"))
         self.query.add(Term("field", "two"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length(), "in order")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits, "in order")
 
         # Ensures slop of 1 does not work for phrases out of order
         # must be at least 2.
@@ -90,8 +91,8 @@
         self.query.setSlop(1)
         self.query.add(Term("field", "two"))
         self.query.add(Term("field", "one"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(0, hits.length(), "reversed, slop not 2 or more")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(0, topDocs.totalHits, "reversed, slop not 2 or more")
 
     def testOrderDoesntMatter(self):
         """
@@ -101,15 +102,15 @@
         self.query.setSlop(2) # must be at least two for reverse order match
         self.query.add(Term("field", "two"))
         self.query.add(Term("field", "one"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length(), "just sloppy enough")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits, "just sloppy enough")
 
         self.query = PhraseQuery()
         self.query.setSlop(2)
         self.query.add(Term("field", "three"))
         self.query.add(Term("field", "one"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(0, hits.length(), "not sloppy enough")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(0, topDocs.totalHits, "not sloppy enough")
 
     def testMulipleTerms(self):
         """
@@ -121,135 +122,138 @@
         self.query.add(Term("field", "one"))
         self.query.add(Term("field", "three"))
         self.query.add(Term("field", "five"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length(), "two total moves")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits, "two total moves")
 
         self.query = PhraseQuery()
         self.query.setSlop(5) # it takes six moves to match this phrase
         self.query.add(Term("field", "five"))
         self.query.add(Term("field", "three"))
         self.query.add(Term("field", "one"))
-        hits = self.searcher.search(self.query)
-        self.assertEqual(0, hits.length(), "slop of 5 not close enough")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(0, topDocs.totalHits, "slop of 5 not close enough")
 
         self.query.setSlop(6)
-        hits = self.searcher.search(self.query)
-        self.assertEqual(1, hits.length(), "slop of 6 just right")
+        topDocs = self.searcher.search(self.query, 50)
+        self.assertEqual(1, topDocs.totalHits, "slop of 6 just right")
 
     def testPhraseQueryWithStopAnalyzer(self):
 
         directory = RAMDirectory()
         stopAnalyzer = StopAnalyzer()
-        writer = IndexWriter(directory, stopAnalyzer, True)
+        writer = IndexWriter(directory, stopAnalyzer, True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         doc = Document()
         doc.add(Field("field", "the stop words are here",
-                      Field.Store.YES, Field.Index.TOKENIZED))
+                      Field.Store.YES, Field.Index.ANALYZED))
         writer.addDocument(doc)
         writer.close()
 
-        searcher = IndexSearcher(directory)
+        searcher = IndexSearcher(directory, True)
 
         # valid exact phrase query
         query = PhraseQuery()
         query.add(Term("field","stop"))
         query.add(Term("field","words"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits)
 
         # currently StopAnalyzer does not leave "holes", so this matches.
         query = PhraseQuery()
         query.add(Term("field", "words"))
         query.add(Term("field", "here"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length())
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits)
 
         searcher.close()
   
     def testPhraseQueryInConjunctionScorer(self):
 
         directory = RAMDirectory()
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
     
         doc = Document()
         doc.add(Field("source", "marketing info",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         writer.addDocument(doc)
     
         doc = Document()
         doc.add(Field("contents", "foobar",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         doc.add(Field("source", "marketing info",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         writer.addDocument(doc)
     
         writer.optimize()
         writer.close()
     
-        searcher = IndexSearcher(directory)
+        searcher = IndexSearcher(directory, True)
     
         phraseQuery = PhraseQuery()
         phraseQuery.add(Term("source", "marketing"))
         phraseQuery.add(Term("source", "info"))
-        hits = searcher.search(phraseQuery)
-        self.assertEqual(2, hits.length())
+        topDocs = searcher.search(phraseQuery, 50)
+        self.assertEqual(2, topDocs.totalHits)
     
         termQuery = TermQuery(Term("contents","foobar"))
         booleanQuery = BooleanQuery()
         booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
         booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
-        hits = searcher.search(booleanQuery)
-        self.assertEqual(1, hits.length())
+        topDocs = searcher.search(booleanQuery, 50)
+        self.assertEqual(1, topDocs.totalHits)
     
         searcher.close()
     
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         doc = Document()
         doc.add(Field("contents", "map entry woo",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         writer.addDocument(doc)
 
         doc = Document()
         doc.add(Field("contents", "woo map entry",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         writer.addDocument(doc)
 
         doc = Document()
         doc.add(Field("contents", "map foobarword entry woo",
-                      Field.Store.YES, Field.Index.TOKENIZED,
+                      Field.Store.YES, Field.Index.ANALYZED,
                       Field.TermVector.YES))
         writer.addDocument(doc)
 
         writer.optimize()
         writer.close()
     
-        searcher = IndexSearcher(directory)
+        searcher = IndexSearcher(directory, True)
     
         termQuery = TermQuery(Term("contents", "woo"))
         phraseQuery = PhraseQuery()
         phraseQuery.add(Term("contents", "map"))
         phraseQuery.add(Term("contents", "entry"))
     
-        hits = searcher.search(termQuery)
-        self.assertEqual(3, hits.length())
-        hits = searcher.search(phraseQuery)
-        self.assertEqual(2, hits.length())
+        topDocs = searcher.search(termQuery, 50)
+        self.assertEqual(3, topDocs.totalHits)
+        topDocs = searcher.search(phraseQuery, 50)
+        self.assertEqual(2, topDocs.totalHits)
     
         booleanQuery = BooleanQuery()
         booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
         booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
-        hits = searcher.search(booleanQuery)
-        self.assertEqual(2, hits.length())
+        topDocs = searcher.search(booleanQuery, 50)
+        self.assertEqual(2, topDocs.totalHits)
     
         booleanQuery = BooleanQuery()
         booleanQuery.add(phraseQuery, BooleanClause.Occur.MUST)
         booleanQuery.add(termQuery, BooleanClause.Occur.MUST)
-        hits = searcher.search(booleanQuery)
-        self.assertEqual(2, hits.length())
+        topDocs = searcher.search(booleanQuery, 50)
+        self.assertEqual(2, topDocs.totalHits)
     
         searcher.close()
         directory.close()
@@ -257,7 +261,7 @@
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_PositionIncrement.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PositionIncrement.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PositionIncrement.py (original)
+++ lucene/pylucene/trunk/test/test_PositionIncrement.py Tue Oct 13 07:38:10 2009
@@ -24,20 +24,26 @@
     def testSetPosition(self):
 
         class _analyzer(PythonAnalyzer):
-            def tokenStream(self, fieldName, reader):
+            def tokenStream(_self, fieldName, reader):
                 class _tokenStream(PythonTokenStream):
-                    def __init__(self):
-                        super(_tokenStream, self).__init__()
-                        self.TOKENS = ["1", "2", "3", "4", "5"]
-                        self.INCREMENTS = [1, 2, 1, 0, 1]
-                        self.i = 0
-                    def next(self):
-                        if self.i == len(self.TOKENS):
-                            return None
-                        t = Token(self.TOKENS[self.i], self.i, self.i)
-                        t.setPositionIncrement(self.INCREMENTS[self.i])
-                        self.i += 1
-                        return t
+                    def __init__(self_):
+                        super(_tokenStream, self_).__init__()
+                        self_.TOKENS = ["1", "2", "3", "4", "5"]
+                        self_.INCREMENTS = [1, 2, 1, 0, 1]
+                        self_.i = 0
+                        self_.posIncrAtt = self_.addAttribute(PositionIncrementAttribute.class_)
+                        self_.termAtt = self_.addAttribute(TermAttribute.class_)
+                        self_.offsetAtt = self_.addAttribute(OffsetAttribute.class_)
+                    def incrementToken(self_):
+                        if self_.i == len(self_.TOKENS):
+                            return False
+                        self_.termAtt.setTermBuffer(self_.TOKENS[self_.i])
+                        self_.offsetAtt.setOffset(self_.i, self_.i)
+                        self_.posIncrAtt.setPositionIncrement(self_.INCREMENTS[self_.i])
+                        self_.i += 1
+                        return True
+                    def end(self):
+                        pass
                     def reset(self):
                         pass
                     def close(self):
@@ -47,76 +53,286 @@
         analyzer = _analyzer()
 
         store = RAMDirectory()
-        writer = IndexWriter(store, analyzer, True)
+        writer = IndexWriter(store, analyzer, True, 
+                             IndexWriter.MaxFieldLength.LIMITED)
         d = Document()
         d.add(Field("field", "bogus",
-                    Field.Store.YES, Field.Index.TOKENIZED))
+                    Field.Store.YES, Field.Index.ANALYZED))
         writer.addDocument(d)
         writer.optimize()
         writer.close()
 
-        searcher = IndexSearcher(store)
+        searcher = IndexSearcher(store, True)
 
+        pos = searcher.getIndexReader().termPositions(Term("field", "1"))
+        pos.next()
+        # first token should be at position 0
+        self.assertEqual(0, pos.nextPosition())
+    
+        pos = searcher.getIndexReader().termPositions(Term("field", "2"))
+        pos.next()
+        # second token should be at position 2
+        self.assertEqual(2, pos.nextPosition())
+    
         q = PhraseQuery()
         q.add(Term("field", "1"))
         q.add(Term("field", "2"))
-        hits = searcher.search(q)
-        self.assertEqual(0, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # same as previous, just specify positions explicitely.
+        q = PhraseQuery() 
+        q.add(Term("field", "1"), 0)
+        q.add(Term("field", "2"), 1)
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # specifying correct positions should find the phrase.
+        q = PhraseQuery()
+        q.add(Term("field", "1"), 0)
+        q.add(Term("field", "2"), 2)
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "2"))
         q.add(Term("field", "3"))
-        hits = searcher.search(q)
-        self.assertEqual(1, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "3"))
         q.add(Term("field", "4"))
-        hits = searcher.search(q)
-        self.assertEqual(0, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # phrase query would find it when correct positions are specified. 
+        q = PhraseQuery()
+        q.add(Term("field", "3"), 0)
+        q.add(Term("field", "4"), 0)
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
+
+        # phrase query should fail for non existing searched term 
+        # even if there exist another searched terms in the same searched
+        # position.
+        q = PhraseQuery()
+        q.add(Term("field", "3"), 0)
+        q.add(Term("field", "9"), 0)
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # multi-phrase query should succed for non existing searched term
+        # because there exist another searched terms in the same searched
+        # position.
+
+        mq = MultiPhraseQuery()
+        mq.add([Term("field", "3"), Term("field", "9")], 0)
+        hits = searcher.search(mq, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "2"))
         q.add(Term("field", "4"))
-        hits = searcher.search(q)
-        self.assertEqual(1, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "3"))
         q.add(Term("field", "5"))
-        hits = searcher.search(q)
-        self.assertEqual(1, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "4"))
         q.add(Term("field", "5"))
-        hits = searcher.search(q)
-        self.assertEqual(1, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
 
         q = PhraseQuery()
         q.add(Term("field", "2"))
         q.add(Term("field", "5"))
-        hits = searcher.search(q)
-        self.assertEqual(0, hits.length())
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # should not find "1 2" because there is a gap of 1 in the index
+        qp = QueryParser("field", StopWhitespaceAnalyzer(False))
+        q = PhraseQuery.cast_(qp.parse("\"1 2\""))
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # omitted stop word cannot help because stop filter swallows the
+        # increments.
+        q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # query parser alone won't help, because stop filter swallows the
+        # increments.
+        qp.setEnablePositionIncrements(True)
+        q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+
+        # stop filter alone won't help, because query parser swallows the
+        # increments.
+        qp.setEnablePositionIncrements(False)
+        q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(0, len(hits))
+      
+        # when both qp qnd stopFilter propagate increments, we should find
+        # the doc.
+        qp = QueryParser("field", StopWhitespaceAnalyzer(True))
+        qp.setEnablePositionIncrements(True)
+        q = PhraseQuery.cast_(qp.parse("\"1 stop 2\""))
+        hits = searcher.search(q, None, 1000).scoreDocs
+        self.assertEqual(1, len(hits))
+
+    def testPayloadsPos0(self):
+
+        for x in xrange(2):
+            dir = RAMDirectory()
+            writer = IndexWriter(dir, TestPayloadAnalyzer(), True,
+                                 IndexWriter.MaxFieldLength.LIMITED)
+            if x == 1:
+                writer.setAllowMinus1Position()
+
+            doc = Document()
+            doc.add(Field("content",
+                          StringReader("a a b c d e a f g h i j a b k k")))
+            writer.addDocument(doc)
+
+            r = writer.getReader()
+
+            tp = r.termPositions(Term("content", "a"))
+            count = 0
+            self.assert_(tp.next())
+            # "a" occurs 4 times
+            self.assertEqual(4, tp.freq())
+
+            if x == 1:
+                expected = Integer.MAX_VALUE
+            else:
+                expected = 0
+
+            self.assertEqual(expected, tp.nextPosition())
+            if x == 1:
+                continue
+
+            self.assertEqual(1, tp.nextPosition())
+            self.assertEqual(3, tp.nextPosition())
+            self.assertEqual(6, tp.nextPosition())
+
+            # only one doc has "a"
+            self.assert_(not tp.next())
+
+            searcher = IndexSearcher(r)
+    
+            stq1 = SpanTermQuery(Term("content", "a"))
+            stq2 = SpanTermQuery(Term("content", "k"))
+            sqs = [stq1, stq2]
+            snq = SpanNearQuery(sqs, 30, False)
+
+            count = 0
+            sawZero = False
+
+            pspans = snq.getSpans(searcher.getIndexReader())
+            while pspans.next():
+                payloads = pspans.getPayload()
+                sawZero |= pspans.start() == 0
+
+                it = payloads.iterator()
+                while it.hasNext():
+                    count += 1
+                    it.next()
+
+            self.assertEqual(5, count)
+            self.assert_(sawZero)
+
+            spans = snq.getSpans(searcher.getIndexReader())
+            count = 0
+            sawZero = False
+            while spans.next():
+                count += 1
+                sawZero |= spans.start() == 0
+
+            self.assertEqual(4, count)
+            self.assert_(sawZero)
+		
+            sawZero = False
+            psu = PayloadSpanUtil(searcher.getIndexReader())
+            pls = psu.getPayloadsForQuery(snq)
+            count = pls.size()
+            it = pls.iterator()
+            while it.hasNext():
+                s = str(String(JArray('byte').cast_(it.next())))
+                sawZero |= s == "pos: 0"
+
+            self.assertEqual(5, count)
+            self.assert_(sawZero)
+            writer.close()
+            searcher.getIndexReader().close()
+            dir.close()
+
+
+class StopWhitespaceAnalyzer(PythonAnalyzer):
+
+    def __init__(self, enablePositionIncrements):
+        super(StopWhitespaceAnalyzer, self).__init__()
+
+        self.enablePositionIncrements = enablePositionIncrements
+        self.a = WhitespaceAnalyzer()
+
+    def tokenStream(self, fieldName, reader):
+
+        ts = self.a.tokenStream(fieldName, reader)
+        return StopFilter(self.enablePositionIncrements, ts, ["stop"])
+
+
+class TestPayloadAnalyzer(PythonAnalyzer):
+
+    def tokenStream(self, fieldName, reader):
+
+        result = LowerCaseTokenizer(reader)
+        return PayloadFilter(result, fieldName)
+
+
+class PayloadFilter(PythonTokenFilter):
+
+    def __init__(self, input, fieldName):
+        super(PayloadFilter, self).__init__(input)
+        self.input = input
+
+        self.fieldName = fieldName
+        self.pos = 0
+        self.i = 0
+        self.posIncrAttr = input.addAttribute(PositionIncrementAttribute.class_)
+        self.payloadAttr = input.addAttribute(PayloadAttribute.class_)
+        self.termAttr = input.addAttribute(TermAttribute.class_)
+
+    def incrementToken(self):
+
+        if self.input.incrementToken():
+            bytes = JArray('byte')("pos: %d" %(self.pos))
+            self.payloadAttr.setPayload(Payload(bytes))
+
+            if self.i % 2 == 1:
+                posIncr = 1
+            else:
+                posIncr = 0
+
+            self.posIncrAttr.setPositionIncrement(posIncr)
+            self.pos += posIncr
+            self.i += 1
+            return True
 
-    def testIncrementingPositions(self):
-        """
-        Basic analyzer behavior should be to keep sequential terms in one
-        increment from one another.
-        """
-        
-        analyzer = WhitespaceAnalyzer()
-        ts = analyzer.tokenStream("field",
-                                  StringReader(u"one two three four five"))
-
-        for token in ts:
-            self.assertEqual(1, token.getPositionIncrement(),
-                             token.termText())
+        return False
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_PrefixFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PrefixFilter.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PrefixFilter.py (original)
+++ lucene/pylucene/trunk/test/test_PrefixFilter.py Tue Oct 13 07:38:10 2009
@@ -30,12 +30,13 @@
                       "/Computers/Mac/Two",
                       "/Computers/Windows"]
 
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
 
         for category in categories:
             doc = Document()
             doc.add(Field("category", category,
-                          Field.Store.YES, Field.Index.UN_TOKENIZED))
+                          Field.Store.YES, Field.Index.NOT_ANALYZED))
             writer.addDocument(doc)
 
         writer.close()
@@ -43,63 +44,63 @@
         # PrefixFilter combined with ConstantScoreQuery
         filter = PrefixFilter(Term("category", "/Computers"))
         query = ConstantScoreQuery(filter)
-        searcher = IndexSearcher(directory)
-        hits = searcher.search(query)
-        self.assertEqual(4, hits.length(),
+        searcher = IndexSearcher(directory, True)
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(4, topDocs.totalHits,
                          "All documents in /Computers category and below")
 
         # test middle of values
         filter = PrefixFilter(Term("category", "/Computers/Mac"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(2, hits.length(), "Two in /Computers/Mac")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(2, topDocs.totalHits, "Two in /Computers/Mac")
 
         # test start of values
         filter = PrefixFilter(Term("category", "/Computers/Linux"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length(), "One in /Computers/Linux")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits, "One in /Computers/Linux")
 
         # test end of values
         filter = PrefixFilter(Term("category", "/Computers/Windows"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length(), "One in /Computers/Windows")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits, "One in /Computers/Windows")
 
         # test non-existant
         filter = PrefixFilter(Term("category", "/Computers/ObsoleteOS"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length(), "no documents")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits, "no documents")
 
         # test non-existant, before values
         filter = PrefixFilter(Term("category", "/Computers/AAA"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length(), "no documents")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits, "no documents")
 
         # test non-existant, after values
         filter = PrefixFilter(Term("category", "/Computers/ZZZ"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length(), "no documents")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits, "no documents")
 
         # test zero-length prefix
         filter = PrefixFilter(Term("category", ""))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(4, hits.length(), "all documents")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(4, topDocs.totalHits, "all documents")
 
         # test non-existant field
         filter = PrefixFilter(Term("nonexistantfield", "/Computers"))
         query = ConstantScoreQuery(filter)
-        hits = searcher.search(query)
-        self.assertEqual(0, hits.length(), "no documents")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(0, topDocs.totalHits, "no documents")
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:

Modified: lucene/pylucene/trunk/test/test_PrefixQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_PrefixQuery.py?rev=824626&r1=824625&r2=824626&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_PrefixQuery.py (original)
+++ lucene/pylucene/trunk/test/test_PrefixQuery.py Tue Oct 13 07:38:10 2009
@@ -26,29 +26,30 @@
         directory = RAMDirectory()
 
         categories = ["/Computers", "/Computers/Mac", "/Computers/Windows"]
-        writer = IndexWriter(directory, WhitespaceAnalyzer(), True)
+        writer = IndexWriter(directory, WhitespaceAnalyzer(), True,
+                             IndexWriter.MaxFieldLength.LIMITED)
         for category in categories:
             doc = Document()
             doc.add(Field("category", category,
-                          Field.Store.YES, Field.Index.UN_TOKENIZED))
+                          Field.Store.YES, Field.Index.NOT_ANALYZED))
             writer.addDocument(doc)
 
         writer.close()
 
         query = PrefixQuery(Term("category", "/Computers"))
-        searcher = IndexSearcher(directory)
-        hits = searcher.search(query)
-        self.assertEqual(3, hits.length(),
+        searcher = IndexSearcher(directory, True)
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(3, topDocs.totalHits,
                          "All documents in /Computers category and below")
 
         query = PrefixQuery(Term("category", "/Computers/Mac"))
-        hits = searcher.search(query)
-        self.assertEqual(1, hits.length(), "One in /Computers/Mac")
+        topDocs = searcher.search(query, 50)
+        self.assertEqual(1, topDocs.totalHits, "One in /Computers/Mac")
 
 
 if __name__ == "__main__":
     import sys, lucene
-    lucene.initVM(lucene.CLASSPATH)
+    lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True: