You are viewing a plain text version of this content. The canonical link for it is here.
Posted to pylucene-commits@lucene.apache.org by va...@apache.org on 2011/02/03 21:20:00 UTC
svn commit: r1066943 - in /lucene/pylucene/trunk: ./
java/org/apache/pylucene/analysis/ java/org/apache/pylucene/search/
java/org/apache/pylucene/store/
samples/LuceneInAction/lia/extsearch/collector/
samples/LuceneInAction/lia/extsearch/filters/ sampl...
Author: vajda
Date: Thu Feb 3 20:20:00 2011
New Revision: 1066943
URL: http://svn.apache.org/viewvc?rev=1066943&view=rev
Log:
accumulated changes to make trunk build again
Removed:
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarityDelegator.java
lucene/pylucene/trunk/test/test_BooleanPrefixQuery.py
Modified:
lucene/pylucene/trunk/CHANGES
lucene/pylucene/trunk/Makefile
lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonCharTokenizer.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py
lucene/pylucene/trunk/test/test_Analyzers.py
lucene/pylucene/trunk/test/test_Binary.py
lucene/pylucene/trunk/test/test_BinaryDocument.py
lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
lucene/pylucene/trunk/test/test_DocBoost.py
lucene/pylucene/trunk/test/test_FilteredQuery.py
lucene/pylucene/trunk/test/test_Similarity.py
lucene/pylucene/trunk/test/test_Sort.py
lucene/pylucene/trunk/test/test_ThaiAnalyzer.py
Modified: lucene/pylucene/trunk/CHANGES
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/CHANGES?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/CHANGES (original)
+++ lucene/pylucene/trunk/CHANGES Thu Feb 3 20:20:00 2011
@@ -9,6 +9,7 @@ Version 3.0.0 ->
- PyLucene built with JCC 2.6
- fixed "Lucene in Action" samples leftover on old API
- improved support for adding optional contrib modules
+ - added --package java.util.regex to wrap constructors on PatternAnalyzer
-
Version 2.9.0 -> 3.0.0
Modified: lucene/pylucene/trunk/Makefile
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/Makefile?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/Makefile (original)
+++ lucene/pylucene/trunk/Makefile Thu Feb 3 20:20:00 2011
@@ -57,16 +57,9 @@ MODULES=lucene-modules-$(LUCENE_VER)
#PREFIX_PYTHON=/opt/local
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
-#JCC=$(PYTHON) -m jcc --arch x86_64
+#JCC=$(PYTHON) -m jcc --shared --arch x86_64
#NUM_FILES=3
-# Mac OS X 10.6 (MacPorts 1.8.0 64-bit Python 2.7, Java 1.6)
-PREFIX_PYTHON=/Users/vajda/apache/pylucene/_install
-ANT=ant
-PYTHON=$(PREFIX_PYTHON)/bin/python
-JCC=$(PYTHON) -m jcc --arch x86_64
-NUM_FILES=3
-
# Mac OS X 10.6 (64-bit and 32-bit Python 2.6 together, Java 1.6)
#PREFIX_PYTHON=/usr
#ANT=ant
@@ -258,6 +251,7 @@ GENERATE=$(JCC) $(foreach jar,$(JARS),--
java.text.SimpleDateFormat \
java.text.DecimalFormat \
java.text.Collator \
+ --package java.util.regex \
--package java.io java.io.StringReader \
java.io.InputStreamReader \
java.io.FileInputStream \
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonCharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonCharTokenizer.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonCharTokenizer.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonCharTokenizer.java Thu Feb 3 20:20:00 2011
@@ -15,18 +15,20 @@
package org.apache.pylucene.analysis;
-import org.apache.lucene.analysis.CharTokenizer;
import java.io.IOException;
import java.io.Reader;
+import org.apache.lucene.analysis.CharTokenizer;
+import org.apache.lucene.util.Version;
+
public class PythonCharTokenizer extends CharTokenizer {
private long pythonObject;
- public PythonCharTokenizer(Reader reader)
+ public PythonCharTokenizer(Version version, Reader reader)
{
- super(reader);
+ super(version, reader);
}
public void pythonExtension(long pythonObject)
@@ -45,6 +47,6 @@ public class PythonCharTokenizer extends
}
public native void pythonDecRef();
- public native boolean isTokenChar(char c);
- public native char normalize(char c);
+ public native boolean isTokenChar(int c);
+ public native int normalize(int c);
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java Thu Feb 3 20:20:00 2011
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
public class PythonCollector extends Collector {
@@ -62,7 +63,7 @@ public class PythonCollector extends Col
public native void pythonDecRef();
public native void collect(int doc, float score)
throws IOException;
- public native void setNextReader(IndexReader reader, int docBase)
+ public native void setNextReader(AtomicReaderContext context)
throws IOException;
public native boolean acceptsDocsOutOfOrder();
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java Thu Feb 3 20:20:00 2011
@@ -18,6 +18,7 @@ package org.apache.pylucene.search;
import java.io.IOException;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
/**
* @author Andi Vajda
@@ -54,7 +55,9 @@ public class PythonFieldComparator exten
throws IOException;
public native void copy(int slot, int doc)
throws IOException;
- public native void setNextReader(IndexReader reader, int docBase)
+
+ public native FieldComparator setNextReader(AtomicReaderContext context)
throws IOException;
+
public native Comparable value(int slot);
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java Thu Feb 3 20:20:00 2011
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
public class PythonFilter extends Filter {
@@ -46,6 +47,6 @@ public class PythonFilter extends Filter
}
public native void pythonDecRef();
- public native DocIdSet getDocIdSet(IndexReader reader)
+ public native DocIdSet getDocIdSet(AtomicReaderContext context)
throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java Thu Feb 3 20:20:00 2011
@@ -16,8 +16,9 @@
package org.apache.pylucene.search;
import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.Term;
import java.util.Collection;
@@ -46,7 +47,7 @@ public class PythonSimilarity extends Si
}
public IDFExplanation idfExplain(final Collection<Term> terms,
- final Searcher searcher)
+ final IndexSearcher searcher)
{
return new IDFExplanation() {
public float getIdf()
@@ -63,11 +64,11 @@ public class PythonSimilarity extends Si
public native void pythonDecRef();
- public native float idfTerms(Collection terms, Searcher searcher);
+ public native float idfTerms(Collection terms, IndexSearcher searcher);
public native float coord(int overlap, int maxOverlap);
public native float idf(int docFreq, int numDocs);
- public native float lengthNorm(String fieldName, int numTokens);
+ public native float computeNorm(String fieldName, FieldInvertState state);
public native float queryNorm(float sumOfSquaredWeights);
public native float sloppyFreq(int distance);
public native float tf(float freq);
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java Thu Feb 3 20:20:00 2011
@@ -16,6 +16,7 @@
package org.apache.pylucene.store;
import java.io.IOException;
+import java.util.Collection;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -28,6 +29,7 @@ public class PythonDirectory extends Dir
private long pythonObject;
public PythonDirectory(LockFactory factory)
+ throws IOException
{
setLockFactory(factory);
}
@@ -47,6 +49,13 @@ public class PythonDirectory extends Dir
pythonDecRef();
}
+ public void sync(Collection<String> names)
+ throws IOException
+ {
+ for (String name : names)
+ sync(name);
+ }
+
public native void pythonDecRef();
public native void close()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/BookLinkCollector.py Thu Feb 3 20:20:00 2011
@@ -31,11 +31,11 @@ class BookLinkCollector(PythonCollector)
return True
- def setNextReader(self, reader, docBase):
+ def setNextReader(self, context):
- self.docBase = docBase
- self.urls = FieldCache.DEFAULT.getTerms(reader, "url")
- self.titles = FieldCache.DEFAULT.getTerms(reader, "title2")
+ self.docBase = context.docBase
+ self.urls = FieldCache.DEFAULT.getTerms(context.reader, "url")
+ self.titles = FieldCache.DEFAULT.getTerms(context.reader, "title2")
def collect(self, docID, score):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilter.py Thu Feb 3 20:20:00 2011
@@ -26,8 +26,9 @@ class SpecialsFilter(PythonFilter):
super(SpecialsFilter, self).__init__()
self.accessor = accessor
- def getDocIdSet(self, reader):
+ def getDocIdSet(self, context):
+ reader = context.reader
bits = OpenBitSet(long(reader.maxDoc()))
isbns = self.accessor.isbns()
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceComparatorSource.py Thu Feb 3 20:20:00 2011
@@ -37,10 +37,12 @@ class DistanceComparatorSource(PythonFie
_self.values = [0.0] * numHits
_self.fieldName = fieldName
- def setNextReader(_self, reader, docBase):
+ def setNextReader(_self, context):
- _self.xDoc = FieldCache.DEFAULT.getInts(reader, "x")
- _self.yDoc = FieldCache.DEFAULT.getInts(reader, "y")
+ _self.xDoc = FieldCache.DEFAULT.getInts(context.reader, "x")
+ _self.yDoc = FieldCache.DEFAULT.getInts(context.reader, "y")
+
+ return _self
def _getDistance(_self, doc):
Modified: lucene/pylucene/trunk/test/test_Analyzers.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Analyzers.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Analyzers.py (original)
+++ lucene/pylucene/trunk/test/test_Analyzers.py Thu Feb 3 20:20:00 2011
@@ -24,7 +24,7 @@ class AnalyzersTestCase(BaseTokenStreamT
def testSimple(self):
- a = SimpleAnalyzer()
+ a = SimpleAnalyzer(Version.LUCENE_CURRENT)
self._assertAnalyzesTo(a, "foo bar FOO BAR",
[ "foo", "bar", "foo", "bar" ])
self._assertAnalyzesTo(a, "foo bar . FOO <> BAR",
@@ -45,7 +45,7 @@ class AnalyzersTestCase(BaseTokenStreamT
def testNull(self):
- a = WhitespaceAnalyzer()
+ a = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
self._assertAnalyzesTo(a, "foo bar FOO BAR",
[ "foo", "bar", "FOO", "BAR" ])
self._assertAnalyzesTo(a, "foo bar . FOO <> BAR",
@@ -85,11 +85,11 @@ class AnalyzersTestCase(BaseTokenStreamT
def testPayloadCopy(self):
s = "how now brown cow"
- ts = WhitespaceTokenizer(StringReader(s))
+ ts = WhitespaceTokenizer(Version.LUCENE_CURRENT, StringReader(s))
ts = PayloadSetter(ts)
self._verifyPayload(ts)
- ts = WhitespaceTokenizer(StringReader(s))
+ ts = WhitespaceTokenizer(Version.LUCENE_CURRENT, StringReader(s))
ts = PayloadSetter(ts)
self._verifyPayload(ts)
Modified: lucene/pylucene/trunk/test/test_Binary.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Binary.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Binary.py (original)
+++ lucene/pylucene/trunk/test/test_Binary.py Thu Feb 3 20:20:00 2011
@@ -22,7 +22,7 @@ class BinaryTestCase(TestCase):
def binary(self, b):
c = JArray('byte')(b)
- field = Field("bin", c, Field.Store.YES)
+ field = Field("bin", c)
v = field.binaryValue
assert c == v and b == [a for a in v]
Modified: lucene/pylucene/trunk/test/test_BinaryDocument.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_BinaryDocument.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_BinaryDocument.py (original)
+++ lucene/pylucene/trunk/test/test_BinaryDocument.py Thu Feb 3 20:20:00 2011
@@ -24,8 +24,7 @@ class TestBinaryDocument(TestCase):
def testBinaryFieldInIndex(self):
bytes = JArray('byte')(self.binaryValStored)
- binaryFldStored = Field("binaryStored", bytes,
- Field.Store.YES)
+ binaryFldStored = Field("binaryStored", bytes)
stringFldStored = Field("stringStored", self.binaryValStored,
Field.Store.YES, Field.Index.NO,
Field.TermVector.NO)
@@ -78,8 +77,8 @@ class TestBinaryDocument(TestCase):
def testCompressionTools(self):
bytes = JArray('byte')(self.binaryValCompressed)
- binaryFldCompressed = Field("binaryCompressed", CompressionTools.compress(bytes), Field.Store.YES)
- stringFldCompressed = Field("stringCompressed", CompressionTools.compressString(self.binaryValCompressed), Field.Store.YES)
+ binaryFldCompressed = Field("binaryCompressed", CompressionTools.compress(bytes))
+ stringFldCompressed = Field("stringCompressed", CompressionTools.compressString(self.binaryValCompressed))
doc = Document()
doc.add(binaryFldCompressed)
Modified: lucene/pylucene/trunk/test/test_CachingWrapperFilter.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_CachingWrapperFilter.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_CachingWrapperFilter.py (original)
+++ lucene/pylucene/trunk/test/test_CachingWrapperFilter.py Thu Feb 3 20:20:00 2011
@@ -29,12 +29,13 @@ class CachingWrapperFilterTestCase(TestC
writer.close()
reader = IndexReader.open(dir, True)
+ context = reader.getTopReaderContext();
class mockFilter(PythonFilter):
def __init__(self):
super(mockFilter, self).__init__()
self._wasCalled = False
- def getDocIdSet(self, reader):
+ def getDocIdSet(self, context):
self._wasCalled = True;
return DocIdBitSet(BitSet())
def clear(self):
@@ -46,12 +47,12 @@ class CachingWrapperFilterTestCase(TestC
cacher = CachingWrapperFilter(filter)
# first time, nested filter is called
- cacher.getDocIdSet(reader)
+ cacher.getDocIdSet(context)
self.assert_(filter.wasCalled(), "first time")
# second time, nested filter should not be called
filter.clear()
- cacher.getDocIdSet(reader)
+ cacher.getDocIdSet(context)
self.assert_(not filter.wasCalled(), "second time")
reader.close()
Modified: lucene/pylucene/trunk/test/test_DocBoost.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_DocBoost.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_DocBoost.py (original)
+++ lucene/pylucene/trunk/test/test_DocBoost.py Thu Feb 3 20:20:00 2011
@@ -59,8 +59,8 @@ class DocBoostTestCase(TestCase):
_self.base = 0
def collect(_self, doc, score):
_self.scores[doc + _self.base] = score
- def setNextReader(_self, reader, docBase):
- _self.base = docBase
+ def setNextReader(_self, context):
+ _self.base = context.docBase
def acceptsDocsOutOfOrder(_self):
return True
Modified: lucene/pylucene/trunk/test/test_FilteredQuery.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_FilteredQuery.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_FilteredQuery.py (original)
+++ lucene/pylucene/trunk/test/test_FilteredQuery.py Thu Feb 3 20:20:00 2011
@@ -66,7 +66,7 @@ class FilteredQueryTestCase(TestCase):
self.query = TermQuery(Term("field", "three"))
class filter(PythonFilter):
- def getDocIdSet(self, reader):
+ def getDocIdSet(self, context):
bitset = BitSet(5)
bitset.set(1)
bitset.set(3)
Modified: lucene/pylucene/trunk/test/test_Similarity.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Similarity.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Similarity.py (original)
+++ lucene/pylucene/trunk/test/test_Similarity.py Thu Feb 3 20:20:00 2011
@@ -77,7 +77,7 @@ class SimilarityTestCase(TestCase):
class collector1(PythonCollector):
def collect(_self, doc, score):
self.assertEqual(1.0, score)
- def setNextReader(_self, reader, docBase):
+ def setNextReader(_self, context):
pass
def acceptsDocsOutOfOrder(_self):
return True
Modified: lucene/pylucene/trunk/test/test_Sort.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Sort.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Sort.py (original)
+++ lucene/pylucene/trunk/test/test_Sort.py Thu Feb 3 20:20:00 2011
@@ -687,7 +687,8 @@ class SortTestCase(TestCase):
# a filter that only allows through the first hit
class filter(PythonFilter):
- def getDocIdSet(_self, reader):
+ def getDocIdSet(_self, context):
+ reader = context.reader
bs = BitSet(reader.maxDoc())
bs.set(0, reader.maxDoc())
bs.set(docs1.scoreDocs[0].doc)
@@ -999,15 +1000,17 @@ class MyFieldComparator(PythonFieldCompa
def setBottom(self, bottom):
self.bottomValue = self.slotValues[bottom]
- def setNextReader(self, reader, docBase):
+ def setNextReader(self, context):
class intParser(PythonIntParser):
def parseInt(_self, val):
return (val.bytes[0] - ord('A')) * 123456
- self.docValues = FieldCache.DEFAULT.getInts(reader, "parser",
+ self.docValues = FieldCache.DEFAULT.getInts(context.reader, "parser",
intParser())
+ return self
+
def value(self, slot):
return Integer(self.slotValues[slot])
@@ -1015,8 +1018,10 @@ class MyFieldComparator(PythonFieldCompa
class MyFieldComparatorSource(PythonFieldComparatorSource):
def newComparator(self, fieldname, numHits, sortPos, reversed):
- return MyFieldComparator(numHits)
-
+ # keep an extra ref since this object seems to be passed around
+ # back and forth without a reference being kept on the java side
+ self.saved = MyFieldComparator(numHits)
+ return self.saved
if __name__ == "__main__":
Modified: lucene/pylucene/trunk/test/test_ThaiAnalyzer.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_ThaiAnalyzer.py?rev=1066943&r1=1066942&r2=1066943&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_ThaiAnalyzer.py (original)
+++ lucene/pylucene/trunk/test/test_ThaiAnalyzer.py Thu Feb 3 20:20:00 2011
@@ -14,15 +14,64 @@
# ====================================================================
from unittest import TestCase, main
-from lucene import ThaiAnalyzer, StringReader, Version
+from lucene import ThaiAnalyzer, ThaiWordFilter, StringReader, Version
from BaseTokenStreamTestCase import BaseTokenStreamTestCase
class ThaiAnalyzerTestCase(BaseTokenStreamTestCase):
- def testAnalyzer(self):
+ def testOffsets(self):
+ self.assert_(ThaiWordFilter.DBBI_AVAILABLE,
+ "JRE does not support Thai dictionary-based BreakIterator")
+
+ self._assertAnalyzesTo(ThaiAnalyzer(Version.LUCENE_CURRENT),
+ u"à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ [ u"à¸à¸²à¸£", u"à¸à¸µà¹", u"à¹à¸à¹", u"à¸à¹à¸à¸", u"à¹à¸ªà¸à¸",
+ u"วà¹à¸²", u"à¸à¸²à¸", u"à¸à¸µ" ],
+ [ 0, 3, 6, 9, 13, 17, 20, 23 ],
+ [ 3, 6, 9, 13, 17, 20, 23, 25 ])
+
+ def testTokenType(self):
+ self.assert_(ThaiWordFilter.DBBI_AVAILABLE,
+ "JRE does not support Thai dictionary-based BreakIterator")
+
+ self._assertAnalyzesTo(ThaiAnalyzer(Version.LUCENE_CURRENT),
+ u"à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ à¹à¹à¹",
+ [ u"à¸à¸²à¸£", u"à¸à¸µà¹", u"à¹à¸à¹", u"à¸à¹à¸à¸", u"à¹à¸ªà¸à¸",
+ u"วà¹à¸²", u"à¸à¸²à¸", u"à¸à¸µ", u"à¹à¹à¹" ],
+ None, None,
+ [ "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
+ "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
+ "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
+ "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
+ "<NUM>" ])
+
+ def testPositionIncrements(self):
+ self.assert_(ThaiWordFilter.DBBI_AVAILABLE,
+ "JRE does not support Thai dictionary-based BreakIterator")
analyzer = ThaiAnalyzer(Version.LUCENE_CURRENT)
+
+ self._assertAnalyzesTo(analyzer, u"à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸ the à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ [ u"à¸à¸²à¸£", u"à¸à¸µà¹", u"à¹à¸à¹", u"à¸à¹à¸à¸", u"à¹à¸ªà¸à¸",
+ u"วà¹à¸²", u"à¸à¸²à¸", u"à¸à¸µ" ],
+ [ 0, 3, 6, 9, 18, 22, 25, 28 ],
+ [ 3, 6, 9, 13, 22, 25, 28, 30 ],
+ None,
+ [ 1, 1, 1, 1, 2, 1, 1, 1 ])
+
+ # case that a stopword is adjacent to thai text, with no whitespace
+ self._assertAnalyzesTo(analyzer, u"à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸the à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ [ u"à¸à¸²à¸£", u"à¸à¸µà¹", u"à¹à¸à¹", u"à¸à¹à¸à¸", u"à¹à¸ªà¸à¸",
+ u"วà¹à¸²", u"à¸à¸²à¸", u"à¸à¸µ" ],
+ [ 0, 3, 6, 9, 17, 21, 24, 27 ],
+ [ 3, 6, 9, 13, 21, 24, 27, 29 ],
+ None,
+ [ 1, 1, 1, 1, 2, 1, 1, 1 ])
+
+ def testAnalyzer30(self):
+
+ analyzer = ThaiAnalyzer(Version.LUCENE_30)
self._assertAnalyzesTo(analyzer, u"", [])