You are viewing a plain text version of this content. The canonical link for it is here.
Posted to pylucene-commits@lucene.apache.org by va...@apache.org on 2012/07/07 17:29:38 UTC
svn commit: r1358592 [1/2] - in /lucene/pylucene/trunk: ./
java/org/apache/pylucene/analysis/ java/org/apache/pylucene/index/
java/org/apache/pylucene/queryParser/
java/org/apache/pylucene/queryParser/classic/
java/org/apache/pylucene/search/ java/org/...
Author: vajda
Date: Sat Jul 7 15:29:35 2012
New Revision: 1358592
URL: http://svn.apache.org/viewvc?rev=1358592&view=rev
Log:
- merged branch 3.x into trunk
- pylucene trunk now tracking lucene's branch_4x
- merged Roman's patch to move build and tests forward
- added makeInterface and makeClass to generate attribute classes on the fly
Added:
lucene/pylucene/trunk/java/org/apache/pylucene/index/
- copied from r1358562, lucene/pylucene/branches/branch_3x/java/org/apache/pylucene/index/
lucene/pylucene/trunk/java/org/apache/pylucene/index/PythonIndexDeletionPolicy.java
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/java/org/apache/pylucene/index/PythonIndexDeletionPolicy.java
lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/
lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonMultiFieldQueryParser.java
- copied, changed from r1358546, lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonMultiFieldQueryParser.java
lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonQueryParser.java
- copied, changed from r1358546, lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonQueryParser.java
lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonList.java
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/java/org/apache/pylucene/util/PythonList.java
lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonListIterator.java
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/java/org/apache/pylucene/util/PythonListIterator.java
lucene/pylucene/trunk/samples/FacetExample.py
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/samples/FacetExample.py
lucene/pylucene/trunk/test/test_Collections.py
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/test/test_Collections.py
lucene/pylucene/trunk/test/test_IndexDeletionPolicy.py
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/test/test_IndexDeletionPolicy.py
lucene/pylucene/trunk/test/test_ReusableAnalyzerBase.py
- copied unchanged from r1358562, lucene/pylucene/branches/branch_3x/test/test_ReusableAnalyzerBase.py
Removed:
lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonMultiFieldQueryParser.java
lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonQueryParser.java
Modified:
lucene/pylucene/trunk/ (props changed)
lucene/pylucene/trunk/CHANGES (contents, props changed)
lucene/pylucene/trunk/Makefile (contents, props changed)
lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonAnalyzer.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonSet.java
lucene/pylucene/trunk/jcc/ (props changed)
lucene/pylucene/trunk/jcc/jcc/cpp.py
lucene/pylucene/trunk/jcc/jcc/sources/JCCEnv.cpp
lucene/pylucene/trunk/jcc/jcc/sources/functions.cpp
lucene/pylucene/trunk/jcc/jcc/sources/jccfuncs.h
lucene/pylucene/trunk/jcc/setup.py (props changed)
lucene/pylucene/trunk/python/collections.py
lucene/pylucene/trunk/samples/ (props changed)
lucene/pylucene/trunk/samples/IndexFiles.py
lucene/pylucene/trunk/samples/LuceneInAction/indexes.tar.gz
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/FilterTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiFieldQueryParserTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiPhraseQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SecurityFilterTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SortingExample.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SpanQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/i18n/ChineseTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/keyword/KeywordAnalyzerTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/positional/PositionalPorterStopAnalyzerTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/WordNetSynonymEngine.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/sorting/DistanceSortingTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/handlingtypes/framework/FileIndexer.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/BaseIndexingTestCase.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/CompoundVersusMultiFileIndexTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentDeleteTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/DocumentUpdateTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FSversusRAMDirectoryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/FieldLengthTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/indexing/VerboseIndexing.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/meetlucene/ (props changed)
lucene/pylucene/trunk/samples/LuceneInAction/lia/meetlucene/Indexer.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BasicSearchingTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/BooleanQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/NumericRangeQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/PrefixQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/QueryParserTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/ScoreTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/searching/TermRangeQueryTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/tools/BerkeleyDbIndexer.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/tools/BerkeleyDbSearcher.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/tools/HighlightTest.py
lucene/pylucene/trunk/samples/LuceneInAction/lia/tools/T9er.py
lucene/pylucene/trunk/samples/SearchFiles.py
lucene/pylucene/trunk/samples/TermPositionVector.py
lucene/pylucene/trunk/samples/manindex.py
lucene/pylucene/trunk/samples/mansearch.py
lucene/pylucene/trunk/test/BaseTestRangeFilter.py
lucene/pylucene/trunk/test/BaseTokenStreamTestCase.py
lucene/pylucene/trunk/test/test_Analyzers.py
lucene/pylucene/trunk/test/test_Binary.py
lucene/pylucene/trunk/test/test_BinaryDocument.py
lucene/pylucene/trunk/test/test_DocBoost.py
lucene/pylucene/trunk/test/test_FilteredQuery.py
lucene/pylucene/trunk/test/test_FuzzyQuery.py
lucene/pylucene/trunk/test/test_Highlighter.py
lucene/pylucene/trunk/test/test_Not.py
lucene/pylucene/trunk/test/test_PhraseQuery.py
lucene/pylucene/trunk/test/test_PositionIncrement.py
lucene/pylucene/trunk/test/test_PrefixFilter.py
lucene/pylucene/trunk/test/test_PrefixQuery.py
lucene/pylucene/trunk/test/test_PyLucene.py
lucene/pylucene/trunk/test/test_PyLuceneThread.py
lucene/pylucene/trunk/test/test_RegexQuery.py
lucene/pylucene/trunk/test/test_Similarity.py
lucene/pylucene/trunk/test/test_Sort.py
lucene/pylucene/trunk/test/test_TermRangeFilter.py
lucene/pylucene/trunk/test/test_TermRangeQuery.py
lucene/pylucene/trunk/test/test_ThaiAnalyzer.py
lucene/pylucene/trunk/test/test_bug1564.py
Propchange: lucene/pylucene/trunk/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Sat Jul 7 15:29:35 2012
@@ -0,0 +1 @@
+^/lucene/pylucene/trunk/jcc jcc
Propchange: lucene/pylucene/trunk/
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/pylucene_3_6:r1334275-1358561
Merged /lucene/pylucene/branches/branch_3x:r943951-944952,944955-984139,984141-984145,984147-1358562
Modified: lucene/pylucene/trunk/CHANGES
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/CHANGES?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/CHANGES (original)
+++ lucene/pylucene/trunk/CHANGES Sat Jul 7 15:29:35 2012
@@ -1,16 +1,53 @@
+Version 3.5.0 -> 3.6.0
+----------------------
+ - using Lucene 3.6.0 sources
+ - renamed classes whose python name would not be unique in lucene module
+ - refreshed Linux build options, added an OpenJDK 7 example
+ - added JavaList to collections.py, a Python java.util.List (Thomas Koch)
+ - added samples/FacetExample.py (Thomas Koch)
+ - PyLucene built with JCC 2.13
+
+Version 3.4 -> 3.5.0
+--------------------
+ - using Lucene 3.5 sources
+ - added facet contrib module to build
+ - refreshed SynonymAnalyzerViewer sample and wordnet index (Thomas Koch)
+ - added PythonReusableAnalyzerBase (Michael McCandless)
+ - added PythonIndexDeletionPolicy.java (Michael McCandless)
+ - added spellchecker contrib module to build
+ - PyLucene built with JCC 2.12
+
+Version 3.3 -> 3.4
+------------------
+ - added new join contrib module to build
+ - PyLucene built with JCC 2.11
-Version 3.0.0 ->
-----------------
+Version 3.2 -> 3.3
+------------------
+ - using Lucene 3.3 sources
+ - adapted to FieldComparator becoming generic
+ - added new grouping contrib module to build
+ - PyLucene built with JCC 2.10
+
+Version 3.1.0 -> 3.2
+--------------------
+ - using Lucene 3.2 sources
+ - PyLucene built with JCC 2.9
+ - rearranged Lucene source checkout tree to reflect new constraints
+
+Version 3.0.0 -> 3.1.0
+----------------------
+ - using Lucene 3.1 sources
- improved support for building on Windows with mingw32
- added wininst target to Makefile
- - added port of ICUNormalizer2Filter using C++ ICU's Normalizer2 via PyICU
- - added port of ICUFoldingFilter using C++ ICU's Normalizer2 via PyICU
- - added port of ICUTransformFilter using C++ ICU's Transliterator via PyICU
- - PyLucene built with JCC 2.6
- - fixed "Lucene in Action" samples leftover on old API
+ - added port of ICUNormalizer2Filter using C++ ICU's Normalizer2 via PyICU 1.1
+ - added port of ICUFoldingFilter using C++ ICU's Normalizer2 via PyICU 1.1
+ - added port of ICUTransformFilter using C++ ICU's Transliterator via PyICU 1.1
+ - fixed "Lucene in Action" samples left over on old API
- improved support for adding optional contrib modules
- added --package java.util.regex to wrap constructors on PatternAnalyzer
- -
+ - fixed mansearch.py sample to reflect API changes
+ - PyLucene built with JCC 2.8
Version 2.9.0 -> 3.0.0
----------------------
Propchange: lucene/pylucene/trunk/CHANGES
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/branch_3x/CHANGES:r943951-944952,944955-1001568,1001570-1358562
Modified: lucene/pylucene/trunk/Makefile
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/Makefile?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/Makefile (original)
+++ lucene/pylucene/trunk/Makefile Sat Jul 7 15:29:35 2012
@@ -18,11 +18,10 @@
VERSION=4.0-0
LUCENE_SVN_VER=HEAD
LUCENE_VER=4.0
-LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/dev/trunk
+LUCENE_SVN=http://svn.apache.org/repos/asf/lucene/dev/branches/branch_4x
PYLUCENE:=$(shell pwd)
LUCENE_SRC=lucene-java-$(LUCENE_VER)
LUCENE=$(LUCENE_SRC)/lucene
-MODULES=$(LUCENE_SRC)/modules
#
# You need to uncomment and edit the variables below in the section
@@ -40,7 +39,7 @@ MODULES=$(LUCENE_SRC)/modules
# - python 2.5:
# $(PYTHON) -m jcc
# - python 2.4:
-# $(PYTHON) $(PREFIX_PYTHON)/lib/python2.4/site-packages/jcc/__init__.py
+# $(PYTHON) $(PREFIX_PYTHON)/lib/python2.4/site-packages/jcc/__main__.py
# NUM_FILES is the number of wrapper files to generate. By default, jcc
# generates all C++ classes into one single file. This may exceed a compiler
# limit.
@@ -51,85 +50,107 @@ MODULES=$(LUCENE_SRC)/modules
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64
-#NUM_FILES=3
+#NUM_FILES=8
# Mac OS X 10.6 (MacPorts 1.8.0 64-bit Python 2.7, Java 1.6)
#PREFIX_PYTHON=/opt/local
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc --shared --arch x86_64
-#NUM_FILES=3
+#NUM_FILES=8
+
+# Mac OS X 10.6 (MacPorts 1.8.0 64-bit Python 2.7, Java 1.6)
+PREFIX_PYTHON=/Users/vajda/apache/pylucene/_install
+ANT=ant
+PYTHON=$(PREFIX_PYTHON)/bin/python
+JCC=$(PYTHON) -m jcc --shared --arch x86_64
+NUM_FILES=8
# Mac OS X 10.6 (64-bit and 32-bit Python 2.6 together, Java 1.6)
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386
-#NUM_FILES=3
+#NUM_FILES=8
# Mac OS X 10.5 (32-bit Python 2.5, Java 1.5)
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc --shared
-#NUM_FILES=3
+#NUM_FILES=8
# Mac OS X (Python 2.3.5, Java 1.5, setuptools 0.6c7, Intel Mac OS X 10.4)
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) /System/Library/Frameworks/Python.framework/Versions/2.3/lib/python2.3/site-packages/JCC-2.3-py2.3-macosx-10.4-i386.egg/jcc/__init__.py
-#NUM_FILES=3
+#NUM_FILES=8
# Mac OS X (Python 2.3.5, Java 1.5, setuptools 0.6c7, PPC Mac OS X 10.4)
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) /System/Library/Frameworks/Python.framework/Versions/2.3/lib/python2.3/site-packages/JCC-2.3-py2.3-macosx-10.4-ppc.egg/jcc/__init__.py
-#NUM_FILES=3
+#NUM_FILES=8
-# Linux (Ubuntu 6.06, Python 2.4, Java 1.5, no setuptools)
+# Linux (Ubuntu 11.10 64-bit, Python 2.7.2, OpenJDK 1.7, setuptools 0.6.16)
+# Be sure to also set JDK['linux2'] in jcc's setup.py to the JAVA_HOME value
+# used below for ANT (and rebuild jcc after changing it).
#PREFIX_PYTHON=/usr
-#ANT=ant
+#ANT=JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 /usr/bin/ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
-#JCC=$(PYTHON) $(PREFIX_PYTHON)/lib/python2.4/site-packages/jcc/__init__.py
-#NUM_FILES=3
+#JCC=$(PYTHON) -m jcc --shared
+#NUM_FILES=8
# Linux (Ubuntu 8.10 64-bit, Python 2.5.2, OpenJDK 1.6, setuptools 0.6c9)
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc --shared
-#NUM_FILES=3
+#NUM_FILES=8
+
+# Linux (Ubuntu 6.06, Python 2.4, Java 1.5, no setuptools)
+#PREFIX_PYTHON=/usr
+#ANT=ant
+#PYTHON=$(PREFIX_PYTHON)/bin/python
+#JCC=$(PYTHON) $(PREFIX_PYTHON)/lib/python2.4/site-packages/jcc/__init__.py
+#NUM_FILES=8
# FreeBSD
#PREFIX_PYTHON=/usr
#ANT=ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) -m jcc
-#NUM_FILES=3
+#NUM_FILES=8
# Solaris (Solaris 11, Python 2.4 32-bit, Sun Studio 12, Java 1.6)
#PREFIX_PYTHON=/usr
#ANT=/usr/local/apache-ant-1.7.0/bin/ant
#PYTHON=$(PREFIX_PYTHON)/bin/python
#JCC=$(PYTHON) $(PREFIX_PYTHON)/lib/python2.4/site-packages/jcc/__init__.py
-#NUM_FILES=3
+#NUM_FILES=8
# Windows (Win32, Python 2.5.1, Java 1.6, ant 1.7.0)
#PREFIX_PYTHON=/cygdrive/o/Python-2.5.2/PCbuild
#ANT=JAVA_HOME=o:\\Java\\jdk1.6.0_02 /cygdrive/o/java/apache-ant-1.7.0/bin/ant
#PYTHON=$(PREFIX_PYTHON)/python.exe
#JCC=$(PYTHON) -m jcc --shared
-#NUM_FILES=3
+#NUM_FILES=8
# Windows (Win32, msys/MinGW, Python 2.6.4, Java 1.6, ant 1.7.1 (WinAnt))
#PREFIX_PYTHON=/c/Python26
#ANT=JAVA_HOME="c:\\Program Files\\Java\\jdk1.6.0_18" "/c/Program Files/WinAnt/bin/ant"
#PYTHON=$(PREFIX_PYTHON)/python.exe
#JCC=$(PYTHON) -m jcc.__main__ --shared --compiler mingw32
-#NUM_FILES=3
+#NUM_FILES=8
+# Windows (Win32, Python 2.7, Java 1.6, ant 1.8.1, Java not on PATH)
+#PREFIX_PYTHON=/cygdrive/c/Python27
+#ANT=JAVA_HOME=c:\\jdk1.6.0_22 /cygdrive/c/java/apache-ant-1.8.1/bin/ant
+#PYTHON=$(PREFIX_PYTHON)/python.exe
+#JCC=$(PYTHON) -m jcc --shared --find-jvm-dll
+#NUM_FILES=8
JARS=$(LUCENE_JAR)
@@ -139,8 +160,14 @@ JARS+=$(MEMORY_JAR) # single
JARS+=$(HIGHLIGHTER_JAR) # needs memory contrib
JARS+=$(EXTENSIONS_JAR) # needs highlighter contrib
JARS+=$(QUERIES_JAR) # regex and other contrib queries
-#JARS+=$(SMARTCN_JAR) # smart chinese analyzer
-#JARS+=$(SPATIAL_JAR) # spatial lucene
+JARS+=$(QUERYPARSER_JAR) # query parser
+JARS+=$(SANDBOX_JAR) # needed by query parser
+#JARS+=$(SMARTCN_JAR) # smart chinese analyzer
+#JARS+=$(SPATIAL_JAR) # spatial lucene
+JARS+=$(GROUPING_JAR) # grouping module
+JARS+=$(JOIN_JAR) # join module
+JARS+=$(FACET_JAR) # facet module
+JARS+=$(SUGGEST_JAR) # suggest/spell module
#
@@ -155,29 +182,37 @@ endif
DEFINES=-DPYLUCENE_VER="\"$(VERSION)\"" -DLUCENE_VER="\"$(LUCENE_VER)\""
-LUCENE_JAR=$(LUCENE)/build/lucene-core-$(LUCENE_VER).jar
-ANALYZERS_JAR=$(MODULES)/analysis/build/common/lucene-analyzers-common-$(LUCENE_VER).jar
-HIGHLIGHTER_JAR=$(LUCENE)/build/contrib/highlighter/lucene-highlighter-$(LUCENE_VER).jar
-MEMORY_JAR=$(LUCENE)/build/contrib/memory/lucene-memory-$(LUCENE_VER).jar
-QUERIES_JAR=$(LUCENE)/build/contrib/queries/lucene-queries-$(LUCENE_VER).jar
+LUCENE_JAR=$(LUCENE)/build/core/lucene-core-$(LUCENE_VER).jar
+ANALYZERS_JAR=$(LUCENE)/build/analysis/common/lucene-analyzers-common-$(LUCENE_VER).jar
+HIGHLIGHTER_JAR=$(LUCENE)/build/highlighter/lucene-highlighter-$(LUCENE_VER).jar
+MEMORY_JAR=$(LUCENE)/build/memory/lucene-memory-$(LUCENE_VER).jar
EXTENSIONS_JAR=build/jar/extensions.jar
-SMARTCN_JAR=$(MODULES)/analysis/build/smartcn/lucene-analyzers-smartcn-$(LUCENE_VER).jar
-SPATIAL_JAR=$(LUCENE)/build/contrib/spatial/lucene-spatial-$(LUCENE_VER).jar
+QUERIES_JAR=$(LUCENE)/build/queries/lucene-queries-$(LUCENE_VER).jar
+QUERYPARSER_JAR=$(LUCENE)/build/queryparser/lucene-queryparser-$(LUCENE_VER).jar
+SANDBOX_JAR=$(LUCENE)/build/sandbox/lucene-sandbox-$(LUCENE_VER).jar
+SMARTCN_JAR=$(LUCENE)/build/analysis/smartcn/lucene-analyzers-smartcn-$(LUCENE_VER).jar
+SPATIAL_JAR=$(LUCENE)/build/spatial/lucene-spatial-$(LUCENE_VER).jar
+GROUPING_JAR=$(LUCENE)/build/grouping/lucene-grouping-$(LUCENE_VER).jar
+JOIN_JAR=$(LUCENE)/build/join/lucene-join-$(LUCENE_VER).jar
+FACET_JAR=$(LUCENE)/build/facet/lucene-facet-$(LUCENE_VER).jar
+SUGGEST_JAR=$(LUCENE)/build/suggest/lucene-suggest-$(LUCENE_VER).jar
ICUPKG:=$(shell which icupkg)
.PHONY: generate compile install default all clean realclean \
- sources test jars distrib
+ sources ivy test jars distrib
default: all
$(LUCENE_SRC):
svn $(SVNOP) --depth files -r $(LUCENE_SVN_VER) $(LUCENE_SVN) $(LUCENE_SRC)
svn $(SVNOP) -r $(LUCENE_SVN_VER) $(LUCENE_SVN)/lucene $(LUCENE_SRC)/lucene
- svn $(SVNOP) -r $(LUCENE_SVN_VER) $(LUCENE_SVN)/modules $(LUCENE_SRC)/modules
sources: $(LUCENE_SRC)
+ivy:
+ cd $(LUCENE); ($(ANT) ivy-fail || $(ANT) ivy-bootstrap)
+
to-orig: sources
mkdir -p $(LUCENE)-orig
tar -C $(LUCENE) -cf - . | tar -C $(LUCENE)-orig -xvf -
@@ -194,25 +229,43 @@ $(LUCENE_JAR): $(LUCENE)
cd $(LUCENE); $(ANT) -Dversion=$(LUCENE_VER)
$(ANALYZERS_JAR): $(LUCENE_JAR)
- cd $(MODULES); $(ANT) -Dversion=$(LUCENE_VER) compile
+ cd $(LUCENE)/analysis; $(ANT) -Dversion=$(LUCENE_VER) compile
$(MEMORY_JAR): $(LUCENE_JAR)
- cd $(LUCENE)/contrib/memory; $(ANT) -Dversion=$(LUCENE_VER)
+ cd $(LUCENE)/memory; $(ANT) -Dversion=$(LUCENE_VER)
$(HIGHLIGHTER_JAR): $(LUCENE_JAR)
- cd $(LUCENE)/contrib/highlighter; $(ANT) -Dversion=$(LUCENE_VER)
+ cd $(LUCENE)/highlighter; $(ANT) -Dversion=$(LUCENE_VER)
$(QUERIES_JAR): $(LUCENE_JAR)
- cd $(LUCENE)/contrib/queries; $(ANT) -Dversion=$(LUCENE_VER)
+ cd $(LUCENE)/queries; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(QUERYPARSER_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/queryparser; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(SANDBOX_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/sandbox; $(ANT) -Dversion=$(LUCENE_VER)
$(EXTENSIONS_JAR): $(LUCENE_JAR)
$(ANT) -f extensions.xml -Dlucene.dir=$(LUCENE_SRC)
$(SMARTCN_JAR): $(LUCENE_JAR)
- cd $(MODULES)/analysis/smartcn; $(ANT) -Dversion=$(LUCENE_VER)
+ cd $(LUCENE)/analysis/smartcn; $(ANT) -Dversion=$(LUCENE_VER)
$(SPATIAL_JAR): $(LUCENE_JAR)
- cd $(LUCENE)/contrib/spatial; $(ANT) -Dversion=$(LUCENE_VER)
+ cd $(LUCENE)/spatial; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(GROUPING_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/grouping; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(JOIN_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/join; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(FACET_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/facet; $(ANT) -Dversion=$(LUCENE_VER)
+
+$(SUGGEST_JAR): $(LUCENE_JAR)
+ cd $(LUCENE)/suggest; $(ANT) -Dversion=$(LUCENE_VER)
JCCFLAGS?=
@@ -221,7 +274,7 @@ jars: $(JARS)
ifneq ($(ICUPKG),)
-ICURES= $(MODULES)/analysis/icu/src/resources
+ICURES= $(LUCENE)/analysis/icu/src/resources
RESOURCES=--resources $(ICURES)
ENDIANNESS:=$(shell $(PYTHON) -c "import struct; print struct.pack('h', 1) == '\000\001' and 'b' or 'l'")
@@ -247,6 +300,9 @@ GENERATE=$(JCC) $(foreach jar,$(JARS),--
--package java.util java.util.Arrays \
java.util.HashMap \
java.util.HashSet \
+ java.lang.IllegalStateException \
+ java.lang.IndexOutOfBoundsException \
+ java.util.NoSuchElementException \
java.text.SimpleDateFormat \
java.text.DecimalFormat \
java.text.Collator \
@@ -254,17 +310,41 @@ GENERATE=$(JCC) $(foreach jar,$(JARS),--
--package java.io java.io.StringReader \
java.io.InputStreamReader \
java.io.FileInputStream \
- --exclude org.apache.lucene.queryParser.Token \
- --exclude org.apache.lucene.queryParser.TokenMgrError \
- --exclude org.apache.lucene.queryParser.QueryParserTokenManager \
- --exclude org.apache.lucene.queryParser.ParseException \
- --exclude org.apache.lucene.search.regex.JakartaRegexpCapabilities \
+ --exclude org.apache.lucene.queryparser.classic.Token \
+ --exclude org.apache.lucene.queryparser.classic.TokenMgrError \
+ --exclude org.apache.lucene.queryparser.classic.QueryParserTokenManager \
+ --exclude org.apache.lucene.queryparser.classic.ParseException \
+ --exclude org.apache.lucene.queryparser.classic.CharStream \
+ --exclude org.apache.lucene.queryparser.classic.FastCharStream \
+ --exclude org.apache.lucene.queryparser.classic.QueryParserConstants \
+ --exclude org.apache.lucene.queryparser.surround.parser.Token \
+ --exclude org.apache.lucene.queryparser.surround.parser.CharStream \
+ --exclude org.apache.lucene.queryparser.surround.parser.QueryParserConstants \
+ --exclude org.apache.lucene.queryparser.surround.parser.TokenMgrError \
+ --exclude org.apache.lucene.queryparser.surround.parser.ParseException \
+ --exclude org.apache.lucene.queryparser.flexible.standard.parser.Token \
+ --exclude org.apache.lucene.queryparser.flexible.standard.parser.TokenMgrError \
+ --exclude org.apache.lucene.queryparser.flexible.standard.parser.ParseException \
+ --exclude org.apache.lucene.sandbox.queries.regex.JakartaRegexpCapabilities \
--exclude org.apache.regexp.RegexpTunnel \
+ --exclude org.apache.lucene.analysis.cn.smart.AnalyzerProfile \
--python lucene \
--mapping org.apache.lucene.document.Document 'get:(Ljava/lang/String;)Ljava/lang/String;' \
--mapping java.util.Properties 'getProperty:(Ljava/lang/String;)Ljava/lang/String;' \
--sequence java.util.AbstractList 'size:()I' 'get:(I)Ljava/lang/Object;' \
--rename org.apache.lucene.search.highlight.SpanScorer=HighlighterSpanScorer \
+ --rename org.apache.lucene.search.highlight.Scorer=HighlighterScorer \
+ --rename org.apache.lucene.search.spell.Dictionary=SpellDictionary \
+ --rename org.apache.lucene.search.suggest.fst.Sort=SuggestSort \
+ --rename org.apache.lucene.store.DataInput=StoreDataInput \
+ --rename org.apache.lucene.store.DataOutput=StoreDataOutput \
+ --rename org.tartarus.snowball.ext.DutchStemmer=DutchPorterStemmer \
+ --rename org.tartarus.snowball.ext.FrenchStemmer=FrenchPorterStemmer \
+ --rename org.tartarus.snowball.ext.GermanStemmer=GermanPorterStemmer \
+ --rename org.tartarus.snowball.ext.PortugueseStemmer=PortuguesePorterStemmer \
+ --rename org.apache.lucene.queryparser.surround.parser.QueryParser=SurroundQueryParser \
+ --rename org.apache.lucene.codecs.lucene40.values.Writer=ValuesWriter \
+ --rename org.apache.lucene.queryparser.xml.QueryBuilder=XMLQueryBuilder \
--version $(LUCENE_VER) \
--module python/collections.py \
--module python/ICUNormalizer2Filter.py \
@@ -288,18 +368,16 @@ bdist: jars
wininst: jars
$(GENERATE) --wininst
-all: sources jars resources compile
+all: sources ivy jars resources compile
@echo build of $(PYLUCENE_LIB) complete
clean:
if test -f $(LUCENE)/build.xml; then cd $(LUCENE); $(ANT) clean; fi
- if test -f $(MODULES)/build.xml; then cd $(MODULES); $(ANT) clean; fi
rm -rf $(LUCENE)/build build
realclean:
if test ! -d $(LUCENE)/.svn; then rm -rf $(LUCENE) lucene; else rm -rf $(LUCENE)/build; fi
- if test ! -d $(MODULES)/.svn; then rm -rf $(MODULES) modules; fi
- rm -rf build samples/LuceneInAction/index
+ rm -rf build samples/LuceneInAction/index samples/LuceneInAction/indexes
OS=$(shell uname)
BUILD_TEST:=$(PYLUCENE)/build/test
@@ -328,17 +406,15 @@ test: install-test samples/LuceneInActio
PYTHONPATH=$(BUILD_TEST) $(PYTHON) samples/LuceneInAction/Explainer.py samples/LuceneInAction/index programming
PYTHONPATH=$(BUILD_TEST) $(PYTHON) samples/LuceneInAction/HighlightIt.py
PYTHONPATH=$(BUILD_TEST) $(PYTHON) samples/LuceneInAction/SortingExample.py
+ PYTHONPATH=$(BUILD_TEST) $(PYTHON) samples/LuceneInAction/SynonymAnalyzerViewer.py
ARCHIVE=pylucene-$(VERSION)-src.tar.gz
-SITE=../site/build/site/en
distrib:
mkdir -p distrib
svn export . distrib/pylucene-$(VERSION)
tar -cf - --exclude build $(LUCENE_SRC) | tar -C distrib/pylucene-$(VERSION) -xvf -
- mkdir distrib/pylucene-$(VERSION)/doc
- tar -C $(SITE) -cf - . | tar -C distrib/pylucene-$(VERSION)/doc -xvf -
cd distrib; tar -cvzf $(ARCHIVE) pylucene-$(VERSION)
cd distrib; gpg2 --armor --output $(ARCHIVE).asc --detach-sig $(ARCHIVE)
cd distrib; openssl md5 < $(ARCHIVE) > $(ARCHIVE).md5
Propchange: lucene/pylucene/trunk/Makefile
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/branch_3x/Makefile:r943951-944952,944955-984139,984141-984145,984147-1001568,1001570-1358562
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonAnalyzer.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonAnalyzer.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/analysis/PythonAnalyzer.java Sat Jul 7 15:29:35 2012
@@ -16,7 +16,6 @@
package org.apache.pylucene.analysis;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
import java.io.Reader;
public class PythonAnalyzer extends Analyzer {
@@ -43,5 +42,5 @@ public class PythonAnalyzer extends Anal
}
public native void pythonDecRef();
- public native TokenStream tokenStream(String fieldName, Reader reader);
+ public native TokenStreamComponents createComponents(final String fieldName, final Reader reader);
}
Copied: lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonMultiFieldQueryParser.java (from r1358546, lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonMultiFieldQueryParser.java)
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonMultiFieldQueryParser.java?p2=lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonMultiFieldQueryParser.java&p1=lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonMultiFieldQueryParser.java&r1=1358546&r2=1358592&rev=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonMultiFieldQueryParser.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonMultiFieldQueryParser.java Sat Jul 7 15:29:35 2012
@@ -13,14 +13,14 @@
* ====================================================================
*/
-package org.apache.pylucene.queryParser;
+package org.apache.pylucene.queryParser.classic;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.Query;
-import org.apache.lucene.queryParser.MultiFieldQueryParser;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
+import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.util.Version;
Copied: lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonQueryParser.java (from r1358546, lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonQueryParser.java)
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonQueryParser.java?p2=lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonQueryParser.java&p1=lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonQueryParser.java&r1=1358546&r2=1358592&rev=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/PythonQueryParser.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/queryParser/classic/PythonQueryParser.java Sat Jul 7 15:29:35 2012
@@ -13,15 +13,15 @@
* ====================================================================
*/
-package org.apache.pylucene.queryParser;
+package org.apache.pylucene.queryParser.classic;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.Query;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.queryParser.CharStream;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.queryparser.classic.CharStream;
+import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.util.Version;
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonCollector.java Sat Jul 7 15:29:35 2012
@@ -19,8 +19,7 @@ import java.io.IOException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.AtomicReaderContext;
public class PythonCollector extends Collector {
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFieldComparator.java Sat Jul 7 15:29:35 2012
@@ -17,14 +17,13 @@ package org.apache.pylucene.search;
import java.io.IOException;
import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.AtomicReaderContext;
/**
* @author Andi Vajda
*/
-public class PythonFieldComparator extends FieldComparator {
+public class PythonFieldComparator<T> extends FieldComparator<T> {
private long pythonObject;
@@ -59,5 +58,8 @@ public class PythonFieldComparator exten
public native FieldComparator setNextReader(AtomicReaderContext context)
throws IOException;
- public native Comparable value(int slot);
+ public native T value(int slot);
+
+ public native int compareDocToValue(int doc, T value)
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonFilter.java Sat Jul 7 15:29:35 2012
@@ -19,8 +19,8 @@ import java.io.IOException;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.index.AtomicReaderContext;
public class PythonFilter extends Filter {
@@ -47,6 +47,6 @@ public class PythonFilter extends Filter
}
public native void pythonDecRef();
- public native DocIdSet getDocIdSet(AtomicReaderContext context)
- throws IOException;
+ public native DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs)
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/search/PythonSimilarity.java Sat Jul 7 15:29:35 2012
@@ -15,11 +15,18 @@
package org.apache.pylucene.search;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.Term;
+
+import java.io.IOException;
import java.util.Collection;
@@ -46,10 +53,10 @@ public class PythonSimilarity extends Si
pythonDecRef();
}
- public IDFExplanation idfExplain(final Collection<Term> terms,
+ public Explanation idfExplain(final Collection<Term> terms,
final IndexSearcher searcher)
{
- return new IDFExplanation() {
+ return new Explanation() {
public float getIdf()
{
return idfTerms(terms, searcher);
@@ -67,14 +74,20 @@ public class PythonSimilarity extends Si
public native float idfTerms(Collection terms, IndexSearcher searcher);
public native float coord(int overlap, int maxOverlap);
- public native float idf(int docFreq, int numDocs);
- public native float computeNorm(FieldInvertState state);
+ public native float idf(long docFreq, long numDocs);
+ public native float computeNorm(String fieldName, FieldInvertState state);
public native float queryNorm(float sumOfSquaredWeights);
public native float sloppyFreq(int distance);
public native float tf(float freq);
public native float scorePayload(int docId, String fieldName,
- int start, int end, byte [] payload,
+ int start, int end, byte[] payload,
int offset, int length);
-
+ // XX: add implementations
+ public native void computeNorm(FieldInvertState state, Norm norm);
+ public native SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats);
+ public native ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context)
+ throws IOException;
+ public native SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context)
+ throws IOException;
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonDirectory.java Sat Jul 7 15:29:35 2012
@@ -19,6 +19,7 @@ import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.LockFactory;
@@ -60,7 +61,7 @@ public class PythonDirectory extends Dir
public native void close()
throws IOException;
- public native IndexOutput createOutput(String name)
+ public native IndexOutput createOutput(String name, IOContext context) //new4.0
throws IOException;
public native void deleteFile(String name)
throws IOException;
@@ -72,12 +73,11 @@ public class PythonDirectory extends Dir
throws IOException;
public native String[] listAll()
throws IOException;
- public native IndexInput openInput(String name)
- throws IOException;
- public native IndexInput openInput(String name, int bufferSize)
+ public native IndexInput openInput(String name, IOContext context) //new4.0
throws IOException;
public native void touchFile(String name)
throws IOException;
public native void sync(String name)
throws IOException;
+
}
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/store/PythonIndexInput.java Sat Jul 7 15:29:35 2012
@@ -17,15 +17,27 @@ package org.apache.pylucene.store;
import java.io.IOException;
import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.IOContext;
public class PythonIndexInput extends BufferedIndexInput {
- private long pythonObject;
+
+ public PythonIndexInput(String resourceDesc) {
+ super(resourceDesc);
+ }
+ public PythonIndexInput(String resourceDesc, int bufferSize) {
+ super(resourceDesc, bufferSize);
+ }
+ public PythonIndexInput(String resourceDesc, IOContext context) {
+ super(resourceDesc, context);
+ }
+
+
+
+
+ private long pythonObject;
- public PythonIndexInput()
- {
- }
public void pythonExtension(long pythonObject)
{
@@ -44,7 +56,7 @@ public class PythonIndexInput extends Bu
public native void pythonDecRef();
- public native Object clone();
+ public native BufferedIndexInput clone();
public native long length();
public native void close()
throws IOException;
Modified: lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonSet.java
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonSet.java?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonSet.java (original)
+++ lucene/pylucene/trunk/java/org/apache/pylucene/util/PythonSet.java Sat Jul 7 15:29:35 2012
@@ -58,12 +58,13 @@ public class PythonSet implements Set {
public native boolean removeAll(Collection c);
public native boolean retainAll(Collection c);
public native int size();
+
public native Object[] toArray();
public Object[] toArray(Object[] a)
{
Object[] array = toArray();
-
+
if (a.length < array.length)
a = (Object[]) Array.newInstance(a.getClass().getComponentType(),
array.length);
Propchange: lucene/pylucene/trunk/jcc/
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/branch_3x/jcc:r943951-944933,944935-944951,944955-944960,944962-945444,945446-945487,945489-945516,945518-945781,945783-984139,984141-984145,984147-1358562
Modified: lucene/pylucene/trunk/jcc/jcc/cpp.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/jcc/jcc/cpp.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/jcc/jcc/cpp.py (original)
+++ lucene/pylucene/trunk/jcc/jcc/cpp.py Sat Jul 7 15:29:35 2012
@@ -95,7 +95,7 @@ PRIMITIVES = { 'boolean': 'Z',
'void': 'V' }
RESERVED = set(['delete', 'and', 'or', 'not', 'xor', 'union', 'register',
- 'const', 'bool', 'operator', 'typeof', 'asm',
+ 'const', 'bool', 'operator', 'typeof', 'asm', 'mutable',
'NULL', 'DOMAIN', 'IGNORE'])
RENAME_METHOD_SUFFIX = '_'
Modified: lucene/pylucene/trunk/jcc/jcc/sources/JCCEnv.cpp
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/jcc/jcc/sources/JCCEnv.cpp?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/jcc/jcc/sources/JCCEnv.cpp (original)
+++ lucene/pylucene/trunk/jcc/jcc/sources/JCCEnv.cpp Sat Jul 7 15:29:35 2012
@@ -123,7 +123,6 @@ void JCCEnv::set_vm(JavaVM *vm, JNIEnv *
vm_env->GetMethodID(_obj, "getClass",
"()Ljava/lang/Class;");
-
jclass iterable = vm_env->FindClass("java/lang/Iterable");
if (iterable == NULL) /* JDK < 1.5 */
@@ -142,7 +141,6 @@ void JCCEnv::set_vm(JavaVM *vm, JNIEnv *
"next", "()Ljava/lang/Object;");
}
-
_mids[mid_enumeration_nextElement] =
vm_env->GetMethodID(vm_env->FindClass("java/util/Enumeration"),
"nextElement", "()Ljava/lang/Object;");
Modified: lucene/pylucene/trunk/jcc/jcc/sources/functions.cpp
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/jcc/jcc/sources/functions.cpp?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/jcc/jcc/sources/functions.cpp (original)
+++ lucene/pylucene/trunk/jcc/jcc/sources/functions.cpp Sat Jul 7 15:29:35 2012
@@ -95,6 +95,170 @@ PyObject *findClass(PyObject *self, PyOb
Py_RETURN_NONE;
}
+static const char interface_bytes[] = {
+ '\xca', '\xfe', '\xba', '\xbe', // magic number: 0xcafebabe
+ '\x00', '\x00', '\x00', '\x32', // version 50.0
+ '\x00', '\x07', // constant pool max index: 6
+ '\x07', '\x00', '\x04', // 1: class name at 4
+ '\x07', '\x00', '\x05', // 2: class name at 5
+ '\x07', '\x00', '\x06', // 3: class name at 6
+ '\x01', '\x00', '\x00', // 4: empty string
+ '\x01', '\x00', '\x10', // 5: 16-byte string: java/lang/Object
+ 'j', 'a', 'v', 'a', '/', 'l', 'a', 'n', 'g', '/',
+ 'O', 'b', 'j', 'e', 'c', 't',
+ '\x01', '\x00', '\x00', // 6: empty string
+ '\x06', '\x01', // public abstract interface
+ '\x00', '\x01', // this class at 1
+ '\x00', '\x02', // superclass at 2
+ '\x00', '\x01', // 1 interface
+ '\x00', '\x03', // interface at 3
+ '\x00', '\x00', // 0 fields
+ '\x00', '\x00', // 0 methods
+ '\x00', '\x00' // 0 attributes
+};
+
+/* make an empty interface that extends an interface */
+PyObject *makeInterface(PyObject *self, PyObject *args)
+{
+ char *name, *extName;
+ int name_len, extName_len;
+
+ if (!PyArg_ParseTuple(args, "s#s#",
+ &name, &name_len, &extName, &extName_len))
+ return NULL;
+
+ JNIEnv *vm_env = env->get_vm_env();
+ jclass _ucl = (jclass) vm_env->FindClass("java/net/URLClassLoader");
+ jmethodID mid = vm_env->GetStaticMethodID(_ucl, "getSystemClassLoader",
+ "()Ljava/lang/ClassLoader;");
+ jobject classLoader = vm_env->CallStaticObjectMethod(_ucl, mid);
+ const int bytes_len = sizeof(interface_bytes);
+ const int len = bytes_len + name_len + extName_len;
+ char *buf = (char *) malloc(len);
+
+ if (buf == NULL)
+ return PyErr_NoMemory();
+
+ int name_pos = 22;
+ int extName_pos = 44;
+ jclass cls;
+
+ memcpy(buf, interface_bytes, name_pos);
+ memcpy(buf + name_pos + name_len, interface_bytes + name_pos,
+ extName_pos - name_pos);
+ memcpy(buf + extName_pos + name_len + extName_len,
+ interface_bytes + extName_pos, bytes_len - extName_pos);
+ extName_pos += name_len;
+
+ *((unsigned short *) (buf + name_pos - 2)) = htons(name_len);
+ memcpy(buf + name_pos, name, name_len);
+
+ *((unsigned short *) (buf + extName_pos - 2)) = htons(extName_len);
+ memcpy(buf + extName_pos, extName, extName_len);
+
+ cls = vm_env->DefineClass(name, classLoader, (const jbyte *) buf, len);
+ free(buf);
+
+ if (cls)
+ return t_Class::wrap_Object(Class(cls));
+
+ return PyErr_SetJavaError();
+}
+
+static const char class_bytes[] = {
+ '\xca', '\xfe', '\xba', '\xbe', // magic number: 0xcafebabe
+ '\x00', '\x00', '\x00', '\x32', // version 50.0
+ '\x00', '\x0c', // constant pool max index: 11
+ '\x0a', '\x00', '\x03', '\x00', '\x08', // 1: method for class 3 at 8
+ '\x07', '\x00', '\x09', // 2: class name at 9
+ '\x07', '\x00', '\x0a', // 3: class name at 10
+ '\x07', '\x00', '\x0b', // 4: class name at 11
+ '\x01', '\x00', '\x06', // 5: 6-byte string: <init>
+ '<', 'i', 'n', 'i', 't', '>',
+ '\x01', '\x00', '\x03', '(', ')', 'V', // 6: 3-byte string: ()V
+ '\x01', '\x00', '\x04', // 7: 4-byte string: Code
+ 'C', 'o', 'd', 'e',
+ '\x0c', '\x00', '\x05', '\x00', '\x06', // 8: name at 5, signature at 6
+ '\x01', '\x00', '\x00', // 9: empty string
+ '\x01', '\x00', '\x00', // 10: empty string
+ '\x01', '\x00', '\x00', // 11: empty string
+ '\x00', '\x21', // super public
+ '\x00', '\x02', // this class at 2
+ '\x00', '\x03', // superclass at 3
+ '\x00', '\x01', // 1 interface
+ '\x00', '\x04', // interface at 4
+ '\x00', '\x00', // 0 fields
+ '\x00', '\x01', // 1 method
+ '\x00', '\x01', '\x00', '\x05', // public, name at 5
+ '\x00', '\x06', '\x00', '\x01', // signature at 6, 1 attribute
+ '\x00', '\x07', // attribute name at 7: Code
+ '\x00', '\x00', '\x00', '\x11', // 17 bytes past 6 attribute bytes
+ '\x00', '\x01', // max stack: 1
+ '\x00', '\x01', // max locals: 1
+ '\x00', '\x00', '\x00', '\x05', // code length: 5
+ '\x2a', '\xb7', '\x00', '\x01', '\xb1', // actual code bytes
+ '\x00', '\x00', // 0 method exceptions
+ '\x00', '\x00', // 0 method attributes
+ '\x00', '\x00', // 0 attributes
+};
+
+/* make an empty class that extends a class and implements an interface */
+PyObject *makeClass(PyObject *self, PyObject *args)
+{
+ char *name, *extName, *implName;
+ int name_len, extName_len, implName_len;
+
+ if (!PyArg_ParseTuple(args, "s#s#s#",
+ &name, &name_len, &extName, &extName_len,
+ &implName, &implName_len))
+ return NULL;
+
+ JNIEnv *vm_env = env->get_vm_env();
+ jclass _ucl = (jclass) vm_env->FindClass("java/net/URLClassLoader");
+ jmethodID mid = vm_env->GetStaticMethodID(_ucl, "getSystemClassLoader",
+ "()Ljava/lang/ClassLoader;");
+ jobject classLoader = vm_env->CallStaticObjectMethod(_ucl, mid);
+ const int bytes_len = sizeof(class_bytes);
+ const int len = bytes_len + name_len + extName_len + implName_len;
+ char *buf = (char *) malloc(len);
+
+ if (buf == NULL)
+ return PyErr_NoMemory();
+
+ int name_pos = 54;
+ int extName_pos = 57;
+ int implName_pos = 60;
+ jclass cls;
+
+ memcpy(buf, class_bytes, name_pos);
+ memcpy(buf + name_pos + name_len, class_bytes + name_pos,
+ extName_pos - name_pos);
+ memcpy(buf + extName_pos + name_len + extName_len,
+ class_bytes + extName_pos, bytes_len - extName_pos);
+ memcpy(buf + implName_pos + name_len + extName_len + implName_len,
+ class_bytes + implName_pos, bytes_len - implName_pos);
+
+ extName_pos += name_len;
+ implName_pos += name_len + extName_len;
+
+ *((unsigned short *) (buf + name_pos - 2)) = htons(name_len);
+ memcpy(buf + name_pos, name, name_len);
+
+ *((unsigned short *) (buf + extName_pos - 2)) = htons(extName_len);
+ memcpy(buf + extName_pos, extName, extName_len);
+
+ *((unsigned short *) (buf + implName_pos - 2)) = htons(implName_len);
+ memcpy(buf + implName_pos, implName, implName_len);
+
+ cls = vm_env->DefineClass(name, classLoader, (const jbyte *) buf, len);
+ free(buf);
+
+ if (cls)
+ return t_Class::wrap_Object(Class(cls));
+
+ return PyErr_SetJavaError();
+}
+
static boxfn get_boxfn(PyTypeObject *type)
{
static PyObject *boxfn_ = PyString_FromString("boxfn_");
Modified: lucene/pylucene/trunk/jcc/jcc/sources/jccfuncs.h
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/jcc/jcc/sources/jccfuncs.h?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/jcc/jcc/sources/jccfuncs.h (original)
+++ lucene/pylucene/trunk/jcc/jcc/sources/jccfuncs.h Sat Jul 7 15:29:35 2012
@@ -23,6 +23,8 @@ PyObject *getVMEnv(PyObject *self);
PyObject *_set_exception_types(PyObject *self, PyObject *args);
PyObject *_set_function_self(PyObject *self, PyObject *args);
PyObject *findClass(PyObject *self, PyObject *args);
+PyObject *makeInterface(PyObject *self, PyObject *args);
+PyObject *makeClass(PyObject *self, PyObject *args);
PyObject *JArray_Type(PyObject *self, PyObject *arg);
PyMethodDef jcc_funcs[] = {
@@ -32,6 +34,10 @@ PyMethodDef jcc_funcs[] = {
METH_NOARGS, NULL },
{ "findClass", (PyCFunction) findClass,
METH_VARARGS, NULL },
+ { "makeInterface", (PyCFunction) makeInterface,
+ METH_VARARGS, NULL },
+ { "makeClass", (PyCFunction) makeClass,
+ METH_VARARGS, NULL },
{ "_set_exception_types", (PyCFunction) _set_exception_types,
METH_VARARGS, NULL },
{ "_set_function_self", (PyCFunction) _set_function_self,
Propchange: lucene/pylucene/trunk/jcc/setup.py
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/branch_3x/jcc/setup.py:r943951-944933,944935-944951,944955-944960,944962-945444,945446-945487,945489-945516,945518-945781,945783-984139,984141-984145,984147-1358562
Modified: lucene/pylucene/trunk/python/collections.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/python/collections.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/python/collections.py (original)
+++ lucene/pylucene/trunk/python/collections.py Sat Jul 7 15:29:35 2012
@@ -10,7 +10,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from lucene import PythonSet, PythonIterator, JavaError
+from lucene import JArray, \
+ PythonSet, PythonList, PythonIterator, PythonListIterator, JavaError, \
+ NoSuchElementException, IllegalStateException, IndexOutOfBoundsException
class JavaSet(PythonSet):
@@ -83,7 +85,7 @@ class JavaSet(PythonSet):
next = _self._iterator.next()
return next
return _iterator()
-
+
def remove(self, obj):
try:
self._set.remove(obj)
@@ -104,7 +106,7 @@ class JavaSet(PythonSet):
def retainAll(self, collection):
result = False
for obj in list(self._set):
- if obj not in c:
+ if obj not in collection:
self._set.remove(obj)
result = True
return result
@@ -112,6 +114,240 @@ class JavaSet(PythonSet):
def size(self):
return len(self._set)
- def toArray(self):
+ def toArray(self): # JavaSet
return list(self._set)
+
+class JavaListIterator(PythonListIterator):
+ """
+ This class implements java.util.ListIterator for a Python list instance it
+ wraps. (simple bidirectional iterator)
+ """
+ def __init__(self, _lst, index=0):
+ super(JavaListIterator, self).__init__()
+ self._lst = _lst
+ self._lastIndex = -1 # keep state for remove/set
+ self.index = index
+
+ def next(self):
+ if self.index >= len(self._lst):
+ raise JavaError, NoSuchElementException(str(self.index))
+ result = self._lst[self.index]
+ self._lastIndex = self.index
+ self.index += 1
+ return result
+
+ def previous(self):
+ if self.index <= 0:
+ raise JavaError, NoSuchElementException(str(self.index - 1))
+ self.index -= 1
+ self._lastIndex = self.index
+ return self._lst[self.index]
+
+ def hasPrevious(self):
+ return self.index > 0
+
+ def hasNext(self):
+ return self.index < len(self._lst)
+
+ def nextIndex(self):
+ return min(self.index, len(self._lst))
+
+ def previousIndex(self):
+ return max(-1, self.index - 1)
+
+ def add(self, element):
+ """
+ Inserts the specified element into the list.
+ The element is inserted immediately before the next element
+ that would be returned by next, if any, and after the next
+ element that would be returned by previous, if any.
+ """
+ if self._lastIndex < 0:
+ raise JavaError, IllegalStateException("add")
+ self._lst.insert(self.index, element)
+ self.index += 1
+ self._lastIndex = -1 # invalidate state
+
+ def remove(self):
+ """
+ Removes from the list the last element that
+ was returned by next or previous.
+ """
+ if self._lastIndex < 0:
+ raise JavaError, IllegalStateException("remove")
+ del self._lst[self._lastIndex]
+ self._lastIndex = -1 # invalidate state
+
+ def set(self, element):
+ """
+ Replaces the last element returned by next or previous
+ with the specified element.
+ """
+ if self._lastIndex < 0:
+ raise JavaError, IllegalStateException("set")
+ self._lst[self._lastIndex] = element
+
+ def __iter__(self):
+ return self
+
+
+class JavaList(PythonList):
+ """
+ This class implements java.util.List around a Python list instance it wraps.
+ """
+
+ def __init__(self, _lst):
+ super(JavaList, self).__init__()
+ self._lst = _lst
+
+ def __contains__(self, obj):
+ return obj in self._lst
+
+ def __len__(self):
+ return len(self._lst)
+
+ def __iter__(self):
+ return iter(self._lst)
+
+ def add(self, index, obj):
+ self._lst.insert(index, obj)
+
+ def addAll(self, collection):
+ size = len(self._lst)
+ self._lst.extend(collection)
+ return len(self._lst) > size
+
+ def addAll(self, index, collection):
+ size = len(self._lst)
+ self._lst[index:index] = collection
+ return len(self._lst) > size
+
+ def clear(self):
+ del self._lst[:]
+
+ def contains(self, obj):
+ return obj in self._lst
+
+ def containsAll(self, collection):
+ for obj in collection:
+ if obj not in self._lst:
+ return False
+ return True
+
+ def equals(self, collection):
+ if type(self) is type(collection):
+ return self._lst == collection._lst
+ return False
+
+ def get(self, index):
+ if index < 0 or index >= self.size():
+ raise JavaError, IndexOutOfBoundsException(str(index))
+ return self._lst[index]
+
+ def indexOf(self, obj):
+ try:
+ return self._lst.index(obj)
+ except ValueError:
+ return -1
+
+ def isEmpty(self):
+ return len(self._lst) == 0
+
+ def iterator(self):
+ class _iterator(PythonIterator):
+ def __init__(_self):
+ super(_iterator, _self).__init__()
+ _self._iterator = iter(self._lst)
+ def hasNext(_self):
+ if hasattr(_self, '_next'):
+ return True
+ try:
+ _self._next = _self._iterator.next()
+ return True
+ except StopIteration:
+ return False
+ def next(_self):
+ if hasattr(_self, '_next'):
+ next = _self._next
+ del _self._next
+ else:
+ next = _self._iterator.next()
+ return next
+ return _iterator()
+
+ def lastIndexOf(self, obj):
+ i = len(self._lst)-1
+ while (i>=0):
+ if obj.equals(self._lst[i]):
+ break
+ i -= 1
+ return i
+
+ def listIterator(self, index=0):
+ return JavaListIterator(self._lst, index)
+
+ def remove(self, obj_or_index):
+ if type(obj_or_index) is type(1):
+ return removeAt(int(obj_or_index))
+ return removeElement(obj_or_index)
+
+ def removeAt(self, pos):
+ """
+ Removes the element at the specified position in this list.
+ Note: private method called from Java via remove(int index)
+ index is already checked (or IndexOutOfBoundsException thrown)
+ """
+ try:
+ el = self._lst[pos]
+ del self._lst[pos]
+ return el
+ except IndexError:
+ # should not happen
+ return None
+
+ def removeObject(self, obj):
+ """
+ Removes the first occurrence of the specified object
+ from this list, if it is present
+ """
+ try:
+ self._lst.remove(obj)
+ return True
+ except ValueError:
+ return False
+
+ def removeAll(self, collection):
+ result = False
+ for obj in collection:
+ if self.removeElement(obj):
+ result = True
+ return result
+
+ def retainAll(self, collection):
+ result = False
+ for obj in self._lst:
+ if obj not in collection and self.removeElement(obj):
+ result = True
+ return result
+
+ def size(self):
+ return len(self._lst)
+
+ def toArray(self):
+ return self._lst
+
+ def subListChecked(self, fromIndex, toIndex):
+ """
+ Note: private method called from Java via subList()
+ from/to index are already checked (or IndexOutOfBoundsException thrown)
+ also IllegalArgumentException is thronw if the endpoint indices
+ are out of order (fromIndex > toIndex)
+ """
+ sublst = self._lst[fromIndex:toIndex]
+ return JavaList(sublst)
+
+ def set(self, index, obj):
+ if index < 0 or index >= self.size():
+ raise JavaError, IndexOutOfBoundsException(str(index))
+ self._lst[index] = obj
Propchange: lucene/pylucene/trunk/samples/
------------------------------------------------------------------------------
Merged /lucene/pylucene/branches/branch_3x/samples:r943951-944952,944955-945799,945801-984139,984141-984145,984147-1358562
Modified: lucene/pylucene/trunk/samples/IndexFiles.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/IndexFiles.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/IndexFiles.py (original)
+++ lucene/pylucene/trunk/samples/IndexFiles.py Sat Jul 7 15:29:35 2012
@@ -31,19 +31,33 @@ class IndexFiles(object):
if not os.path.exists(storeDir):
os.mkdir(storeDir)
store = lucene.SimpleFSDirectory(lucene.File(storeDir))
- writer = lucene.IndexWriter(store, analyzer, True,
- lucene.IndexWriter.MaxFieldLength.LIMITED)
- writer.setMaxFieldLength(1048576)
+ analyzer = lucene.LimitTokenCountAnalyzer(analyzer, 1048576)
+ config = lucene.IndexWriterConfig(lucene.Version.LUCENE_CURRENT, analyzer)
+ writer = lucene.IndexWriter(store, config)
self.indexDocs(root, writer)
ticker = Ticker()
- print 'optimizing index',
+ print 'commit index',
threading.Thread(target=ticker.run).start()
- writer.optimize()
+ writer.commit()
writer.close()
ticker.tick = False
print 'done'
def indexDocs(self, root, writer):
+
+ t1 = lucene.FieldType()
+ t1.setIndexed(True)
+ t1.setStored(True)
+ t1.setTokenized(False)
+ t1.setIndexOptions(lucene.FieldInfo.IndexOptions.DOCS_AND_FREQS)
+
+ t2 = lucene.FieldType()
+ t2.setIndexed(True)
+ t2.setStored(False)
+ t2.setTokenized(True)
+ t2.setIndexOptions(lucene.FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+
+
for root, dirnames, filenames in os.walk(root):
for filename in filenames:
if not filename.endswith('.txt'):
@@ -56,15 +70,12 @@ class IndexFiles(object):
file.close()
doc = lucene.Document()
doc.add(lucene.Field("name", filename,
- lucene.Field.Store.YES,
- lucene.Field.Index.NOT_ANALYZED))
+ t1))
doc.add(lucene.Field("path", path,
- lucene.Field.Store.YES,
- lucene.Field.Index.NOT_ANALYZED))
+ t2))
if len(contents) > 0:
doc.add(lucene.Field("contents", contents,
- lucene.Field.Store.NO,
- lucene.Field.Index.ANALYZED))
+ t2))
else:
print "warning: no content in %s" % filename
writer.addDocument(doc)
@@ -84,3 +95,4 @@ if __name__ == '__main__':
print end - start
except Exception, e:
print "Failed: ", e
+ raise e
Modified: lucene/pylucene/trunk/samples/LuceneInAction/indexes.tar.gz
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/indexes.tar.gz?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/FilterTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/FilterTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/FilterTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/FilterTest.py Sat Jul 7 15:29:35 2012
@@ -28,7 +28,7 @@ class FilterTest(LiaTestCase):
super(FilterTest, self).setUp()
self.allBooks = MatchAllDocsQuery()
- self.searcher = IndexSearcher(self.directory, True)
+ self.searcher = self.getSearcher()
scoreDocs = self.searcher.search(self.allBooks, 50).scoreDocs
self.numAllBooks = len(scoreDocs)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiFieldQueryParserTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiFieldQueryParserTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiFieldQueryParserTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiFieldQueryParserTest.py Sat Jul 7 15:29:35 2012
@@ -29,7 +29,7 @@ class MultiFieldQueryParserTest(LiaTestC
[SHOULD, SHOULD],
SimpleAnalyzer())
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
scoreDocs = searcher.search(query, 50).scoreDocs
self.assertHitsIncludeTitle(searcher, scoreDocs,
@@ -47,7 +47,7 @@ class MultiFieldQueryParserTest(LiaTestC
[MUST, MUST],
SimpleAnalyzer())
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
scoreDocs = searcher.search(query, 50).scoreDocs
self.assertHitsIncludeTitle(searcher, scoreDocs,
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiPhraseQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiPhraseQueryTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiPhraseQueryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/MultiPhraseQueryTest.py Sat Jul 7 15:29:35 2012
@@ -37,7 +37,7 @@ class MultiPhraseQueryTest(TestCase):
writer.addDocument(doc2)
writer.close()
- self.searcher = IndexSearcher(directory, True)
+ self.searcher = self.getSearcher()
def testBasic(self):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SecurityFilterTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SecurityFilterTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SecurityFilterTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SecurityFilterTest.py Sat Jul 7 15:29:35 2012
@@ -48,7 +48,7 @@ class SecurityFilterTest(TestCase):
query = TermQuery(Term("keywords", "info"))
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
topDocs = searcher.search(query, 50)
self.assertEqual(2, topDocs.totalHits, "Both documents match")
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SortingExample.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SortingExample.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SortingExample.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SortingExample.py Sat Jul 7 15:29:35 2012
@@ -28,7 +28,7 @@ class SortingExample(object):
def displayResults(self, query, sort):
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
fillFields = False
computeMaxScore = False
@@ -61,7 +61,7 @@ class SortingExample(object):
print " ", doc["category"]
# print searcher.explain(query, scoreDoc.doc)
- searcher.close()
+ del searcher
def main(cls, argv):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SpanQueryTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SpanQueryTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SpanQueryTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/advsearching/SpanQueryTest.py Sat Jul 7 15:29:35 2012
@@ -46,7 +46,7 @@ class SpanQueryTest(TestCase):
writer.close()
- self.searcher = IndexSearcher(self.directory, True)
+ self.searcher = self.getSearcher()
self.reader = IndexReader.open(self.directory, True)
self.quick = SpanTermQuery(Term("f", "quick"))
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/i18n/ChineseTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/i18n/ChineseTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/i18n/ChineseTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/i18n/ChineseTest.py Sat Jul 7 15:29:35 2012
@@ -21,7 +21,7 @@ class ChineseTest(LiaTestCase):
def testChinese(self):
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
query = TermQuery(Term("contents", "é"))
scoreDocs = searcher.search(query, 50).scoreDocs
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/keyword/KeywordAnalyzerTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/keyword/KeywordAnalyzerTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/keyword/KeywordAnalyzerTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/keyword/KeywordAnalyzerTest.py Sat Jul 7 15:29:35 2012
@@ -40,7 +40,7 @@ class KeywordAnalyzerTest(TestCase):
writer.addDocument(doc)
writer.close()
- self.searcher = IndexSearcher(self.directory, True)
+ self.searcher = self.getSearcher()
def testTermQuery(self):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/positional/PositionalPorterStopAnalyzerTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/positional/PositionalPorterStopAnalyzerTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/positional/PositionalPorterStopAnalyzerTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/positional/PositionalPorterStopAnalyzerTest.py Sat Jul 7 15:29:35 2012
@@ -57,7 +57,7 @@ class PositionalPorterStopAnalyzerTest(T
def testExactPhrase(self):
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
query = QueryParser(Version.LUCENE_24, "contents",
self.porterAnalyzer).parse('"over the lazy"')
topDocs = searcher.search(query, 50)
@@ -66,7 +66,7 @@ class PositionalPorterStopAnalyzerTest(T
def testWithSlop(self):
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
parser = QueryParser(Version.LUCENE_CURRENT, "contents",
self.porterAnalyzer)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/SynonymAnalyzerTest.py Sat Jul 7 15:29:35 2012
@@ -41,11 +41,11 @@ class SynonymAnalyzerTest(TestCase):
writer.addDocument(doc)
writer.close()
- self.searcher = IndexSearcher(self.directory, True)
+ self.searcher = self.getSearcher()
def tearDown(self):
- self.searcher.close()
+ del self.searcher
def testJumps(self):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/WordNetSynonymEngine.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/WordNetSynonymEngine.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/WordNetSynonymEngine.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/analysis/synonym/WordNetSynonymEngine.py Sat Jul 7 15:29:35 2012
@@ -13,22 +13,24 @@
# ====================================================================
from lucene import \
- Document, Term, IndexSearcher, TermQuery, FSDirectory, RAMDirectory, Hit
+ Document, Term, IndexSearcher, TermQuery, \
+ SimpleFSDirectory, RAMDirectory, File
class WordNetSynonymEngine(object):
def __init__(self, indexDir):
- self.directory = RAMDirectory(indexDir)
+ self.directory = RAMDirectory(SimpleFSDirectory(File(indexDir)))
self.searcher = IndexSearcher(self.directory)
def getSynonyms(self, word):
synList = []
-
- for hit in self.searcher.search(TermQuery(Term("word", word))):
- doc = Hit.cast_(hit).getDocument()
+ topDocs = self.searcher.search(TermQuery(Term("word", word)), 50)
+
+ for scoreDoc in topDocs.scoreDocs:
+ doc = self.searcher.doc(scoreDoc.doc)
for value in doc.getValues("syn"):
synList.append(value)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/common/LiaTestCase.py Sat Jul 7 15:29:35 2012
@@ -17,10 +17,14 @@ import os
from unittest import TestCase
from lucene import \
SimpleFSDirectory, Document, \
- System, SimpleDateFormat, File
+ System, SimpleDateFormat, File, IndexSearcher, DirectoryReader, \
+ Version, IndexWriterConfig, LimitTokenCountAnalyzer, WhitespaceAnalyzer, \
+ IndexWriter
class LiaTestCase(TestCase):
+
+ TEST_VERSION = Version.LUCENE_CURRENT
def __init__(self, *args):
@@ -32,9 +36,24 @@ class LiaTestCase(TestCase):
self.directory = SimpleFSDirectory(File(self.indexDir))
def tearDown(self):
-
self.directory.close()
-
+
+
+ def getWriter(self, directory=None, analyzer=None, open_mode=None):
+ config = IndexWriterConfig(self.TEST_VERSION,
+ analyzer or LimitTokenCountAnalyzer(WhitespaceAnalyzer(Version.LUCENE_CURRENT), 10000)
+ )
+ config.setOpenMode(open_mode or IndexWriterConfig.OpenMode.CREATE)
+ return IndexWriter(directory or self.directory, config)
+
+
+ def getSearcher(self, directory=None, reader=None):
+ if reader is not None:
+ return IndexSearcher(reader)
+
+ return IndexSearcher(DirectoryReader.open(directory or self.directory))
+
+
#
# For troubleshooting
#
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/common/TestDataDocumentHandler.py Sat Jul 7 15:29:35 2012
@@ -15,8 +15,10 @@
import os
from lucene import \
- Document, Field, IndexWriter, StandardAnalyzer, NumericField, \
- SimpleDateFormat, Version, SimpleFSDirectory, File, DateTools, DateField
+ Document, Field, IndexWriter, StandardAnalyzer, IntField, \
+ SimpleDateFormat, Version, SimpleFSDirectory, File, DateTools, \
+ IndexWriterConfig, LogMergePolicy, FieldType, TextField, StringField, \
+ FieldInfo
# date culled from LuceneInAction.zip archive from Manning site
samplesModified = SimpleDateFormat('yyyy-MM-dd').parse('2004-12-02')
@@ -27,17 +29,23 @@ class TestDataDocumentHandler(object):
def createIndex(cls, dataDir, indexDir, useCompound):
indexDir = SimpleFSDirectory(File(indexDir))
- writer = IndexWriter(indexDir,
- StandardAnalyzer(Version.LUCENE_CURRENT), True,
- IndexWriter.MaxFieldLength.UNLIMITED)
- writer.setUseCompoundFile(useCompound)
+ config = IndexWriterConfig(Version.LUCENE_CURRENT,
+ StandardAnalyzer(Version.LUCENE_CURRENT))
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+
+ writer = IndexWriter(indexDir, config)
+ config = writer.getConfig()
+ mp = config.getMergePolicy()
+
+ if (LogMergePolicy.instance_(mp)):
+ mp.setUseCompoundFile(useCompound)
for dir, dirnames, filenames in os.walk(dataDir):
for filename in filenames:
if filename.endswith('.properties'):
cls.indexFile(writer, os.path.join(dir, filename), dataDir)
- writer.optimize()
+ writer.commit()
writer.close()
def indexFile(cls, writer, path, baseDir):
@@ -72,46 +80,74 @@ class TestDataDocumentHandler(object):
print category.encode('utf-8')
print "---------"
- doc.add(Field("isbn", isbn,
- Field.Store.YES, Field.Index.NOT_ANALYZED))
- doc.add(Field("category", category,
- Field.Store.YES, Field.Index.NOT_ANALYZED))
- doc.add(Field("title", title,
- Field.Store.YES, Field.Index.ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS))
- doc.add(Field("title2", title.lower(),
- Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS,
- Field.TermVector.WITH_POSITIONS_OFFSETS))
+ doc.add(Field("isbn", isbn, StringField.TYPE_STORED))
+
+ doc.add(Field("category", category, StringField.TYPE_STORED))
+
+ # note: ft should be initialized once and re-used
+ ft = FieldType()
+ ft.setIndexed(True)
+ ft.setTokenized(True)
+ ft.setStored(True)
+ ft.setStoreTermVectorPositions(True)
+ ft.setStoreTermVectorOffsets(True)
+ ft.freeze()
+ doc.add(Field("title", title, ft))
+
+ ft = FieldType(StringField.TYPE_STORED)
+ ft.setIndexed(True)
+ ft.setTokenized(False)
+ ft.setOmitNorms(True)
+ ft.setStoreTermVectorPositions(True)
+ ft.setStoreTermVectorOffsets(True)
+ doc.add(Field("title2", title.lower(), ft))
# split multiple authors into unique field instances
authors = author.split(',')
+ ft = FieldType()
+ ft.setIndexed(True)
+ ft.setTokenized(False)
+ ft.setStored(True)
+ ft.setStoreTermVectorPositions(True)
+ ft.setStoreTermVectorOffsets(True)
for a in authors:
- doc.add(Field("author", a,
- Field.Store.YES, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS))
-
- doc.add(Field("url", url,
- Field.Store.YES,
- Field.Index.NOT_ANALYZED_NO_NORMS))
- doc.add(Field("subject", subject,
- Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS))
- doc.add(NumericField("pubmonth",
- Field.Store.YES,
- True).setIntValue(int(pubmonth)))
+ doc.add(Field("author", a, ft))
+
+ ft = FieldType()
+ ft.setIndexed(True)
+ ft.setTokenized(False)
+ ft.setStored(True)
+ ft.setOmitNorms(True)
+ doc.add(Field("url", url, ft))
+
+ ft = FieldType()
+ ft.setIndexed(True)
+ ft.setTokenized(True)
+ ft.setStored(False)
+ ft.setStoreTermVectorPositions(True)
+ ft.setStoreTermVectorOffsets(True)
+ doc.add(Field("subject", subject, ft))
+
+ doc.add(IntField("pubmonth", int(pubmonth), Field.Store.YES))
d = DateTools.stringToDate(pubmonth)
d = int(d.getTime() / (1000 * 3600 * 24.0))
- doc.add(NumericField("pubmonthAsDay").setIntValue(d))
+ doc.add(IntField("pubmonthAsDay", d, IntField.TYPE_NOT_STORED))
+ ft = FieldType()
+ ft.setIndexed(True)
+ ft.setTokenized(True)
+ ft.setStored(False)
+ ft.setStoreTermVectorPositions(True)
+ ft.setStoreTermVectorOffsets(True)
doc.add(Field("contents", ' '.join([title, subject, author, category]),
- Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS))
+ ft))
doc.add(Field("path", path,
- Field.Store.YES, Field.Index.NOT_ANALYZED))
- doc.add(Field("modified", DateField.dateToString(samplesModified),
- Field.Store.YES, Field.Index.NOT_ANALYZED))
+ StringField.TYPE_STORED))
+
+ doc.add(Field("modified", DateTools.dateToString(samplesModified, DateTools.Resolution.MILLISECOND),
+ StringField.TYPE_STORED))
writer.addDocument(doc)
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/collector/CollectorTest.py Sat Jul 7 15:29:35 2012
@@ -22,7 +22,7 @@ class CollectorTest(LiaTestCase):
def testCollecting(self):
query = TermQuery(Term("contents", "junit"))
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
collector = BookLinkCollector(searcher)
searcher.search(query, collector)
@@ -34,4 +34,4 @@ class CollectorTest(LiaTestCase):
scoreDocs = searcher.search(query, 10).scoreDocs
self.dumpHits(searcher, scoreDocs)
- searcher.close()
+ del searcher
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/filters/SpecialsFilterTest.py Sat Jul 7 15:29:35 2012
@@ -28,7 +28,7 @@ class SpecialsFilterTest(LiaTestCase):
super(SpecialsFilterTest, self).setUp()
self.allBooks = MatchAllDocsQuery()
- self.searcher = IndexSearcher(self.directory, True)
+ self.searcher = self.getSearcher()
def testCustomFilter(self):
Modified: lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py?rev=1358592&r1=1358591&r2=1358592&view=diff
==============================================================================
--- lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py (original)
+++ lucene/pylucene/trunk/samples/LuceneInAction/lia/extsearch/queryparser/AdvancedQueryParserTest.py Sat Jul 7 15:29:35 2012
@@ -85,7 +85,7 @@ class AdvancedQueryParserTest(TestCase):
self.assertEqual("id:[0000000037 TO 0000000346]",
query.toString("field"), "padded")
- searcher = IndexSearcher(self.directory, True)
+ searcher = self.getSearcher()
scoreDocs = searcher.search(query, 1000).scoreDocs
self.assertEqual(310, len(scoreDocs))