You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/01/05 21:25:44 UTC
svn commit: r1055622 [2/14] - in /lucene/dev/branches/docvalues: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/
dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/...
Modified: lucene/dev/branches/docvalues/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/build.xml?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/build.xml (original)
+++ lucene/dev/branches/docvalues/build.xml Wed Jan 5 20:25:17 2011
@@ -52,6 +52,101 @@
</sequential>
</target>
+ <target name="eclipse" description="Setup Eclipse configuration">
+ <copy file="dev-tools/eclipse/dot.classpath" tofile=".classpath" overwrite="true"/>
+ <mkdir dir=".settings"/>
+ <copy file="dev-tools/eclipse/resources.prefs"
+ tofile=".settings/org.eclipse.core.resources.prefs" overwrite="true"/>
+ <echo>Running Lucene contrib db/bdb-je task 'get-je-jar' ...</echo>
+ <subant target="get-je-jar">
+ <fileset dir="lucene/contrib/db/bdb-je" includes="build.xml" />
+ </subant>
+ <echo>Running Lucene contrib db/bdb task 'get-db-jar' ...</echo>
+ <subant target="get-db-jar">
+ <fileset dir="lucene/contrib/db/bdb" includes="build.xml" />
+ </subant>
+ <echo>
+ SUCCESS: You must right-click your project and choose Refresh
+ Please note, your project must use a Java 6 JRE
+ </echo>
+ </target>
+
+ <condition property="isMac"><os family="mac"/></condition>
+ <condition property="isUnix"><os family="unix"/></condition>
+ <condition property="isWindows"><os family="windows"/></condition>
+ <property environment="env"/>
+ <target name="idea" depends="copy-idea-files,finish-idea-setup-windows,
+ finish-idea-setup-mac,
+ finish-idea-setup-unix"
+ description="Setup IntelliJ IDEA configuration files"/>
+ <target name="copy-idea-files">
+ <copy todir=".">
+ <fileset dir="dev-tools/idea">
+ <exclude name="Intellij-Lucene-Codestyle.xml"/>
+ </fileset>
+ </copy>
+ <echo>Running Lucene contrib db/bdb-je task 'get-je-jar' ...</echo>
+ <subant target="get-je-jar">
+ <fileset dir="lucene/contrib/db/bdb-je" includes="build.xml" />
+ </subant>
+ <echo>Running Lucene contrib db/bdb task 'get-db-jar' ...</echo>
+ <subant target="get-db-jar">
+ <fileset dir="lucene/contrib/db/bdb" includes="build.xml" />
+ </subant>
+ </target>
+ <target name="finish-idea-setup-windows" if="isWindows">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools\idea\Intellij-Lucene-Codestyle.xml to
+ ${env.HOMEDRIVE}${env.HOMEPATH}\.IntelliJIdeaXX\config\codestyles\
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="finish-idea-setup-mac" if="isMac">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools/idea/Intellij-Lucene-Codestyle.xml to
+ ~/Library/Preferences/IntelliJXX/codestyles/
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="finish-idea-setup-unix" if="isUnix">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools/idea/Intellij-Lucene-Codestyle.xml to
+ ~/.IntelliJIdeaXX/config/codestyles/
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="clean-idea"
+ description="Removes all IntelliJ IDEA configuration files">
+ <delete dir=".idea" failonerror="true"/>
+ <delete failonerror="true">
+ <fileset dir="." includes="*.iml,*.ipr,*.iws"/>
+ <fileset dir="solr" includes="**/*.iml"/>
+ <fileset dir="lucene" includes="**/*.iml"/>
+ <fileset dir="modules" includes="**/*.iml"/>
+ </delete>
+ </target>
+
<target name="clean" description="Clean Lucene and Solr">
<delete dir="dist" />
<sequential>
Modified: lucene/dev/branches/docvalues/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/CHANGES.txt?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/lucene/CHANGES.txt Wed Jan 5 20:25:17 2011
@@ -119,10 +119,19 @@ Changes in backwards compatibility polic
need to change it (e.g. using "\\" to escape '\' itself).
(Sunil Kamath, Terry Yang via Robert Muir)
+* LUCENE-2771: IndexReader.norms() now throws UOE on non-atomic IndexReaders. If
+ you really want a top-level norms, use MultiNorms or SlowMultiReaderWrapper.
+ (Uwe Schindler, Robert Muir)
+
+* LUCENE-2837: Collapsed Searcher, Searchable into IndexSearcher;
+ removed contrib/remote and MultiSearcher (Mike McCandless); absorbed
+ ParallelMultiSearcher into IndexSearcher as an optional
+ ExecutorServiced passed to its ctor. (Mike McCandless)
+
Changes in Runtime Behavior
-* LUCENE-2650: The behavior of FSDirectory.open has changed. On 64-bit
- Windows systems that support unmapping, FSDirectory.open returns
+* LUCENE-2650, LUCENE-2825: The behavior of FSDirectory.open has changed. On 64-bit
+ Windows and Solaris systems that support unmapping, FSDirectory.open returns
MMapDirectory. Additionally the behavior of MMapDirectory has been
changed to enable unmapping by default if supported by the JRE.
(Mike McCandless, Uwe Schindler, Robert Muir)
@@ -134,7 +143,7 @@ Changes in Runtime Behavior
* LUCENE-2805: IndexWriter now increments the index version on every change to
the index instead of for every commit. Committing or closing the IndexWriter
without any changes to the index will not cause any index version increment.
- (Simon Willnauer, Mike Mccandless)
+ (Simon Willnauer, Mike McCandless)
API Changes
@@ -298,7 +307,7 @@ New features
* LUCENE-995: The QueryParser now interprets * as an open end for range
queries. Literal asterisks may be represented by quoting or escaping
- (i.e. \* or "*") Custom QueryParser sublcasses overriding getRangeQuery()
+ (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery()
will be passed null for any open endpoint. (Adriano Crestani, yonik)
* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now
@@ -311,6 +320,12 @@ New features
extensions within the same segment. Codecs now use their per-segment codec
ID in the file names. (Simon Willnauer)
+* LUCENE-2843: Added a new terms index impl,
+ VariableGapTermsIndexWriter/Reader, that accepts a pluggable
+ IndexTermSelector for picking which terms should be indexed in the
+ terms dict. This impl stores the indexed terms in an FST, which is
+ much more RAM efficient than FixedGapTermsIndex. (Mike McCandless)
+
Optimizations
* LUCENE-2410: ~20% speedup on exact (slop=0) PhraseQuery matching.
@@ -324,7 +339,7 @@ Optimizations
efficient copying by sub-classes. Optimized copy is implemented for RAM and FS
streams. (Shai Erera)
-* LUCENE-2588: Don't store unecessary suffixes when writing the terms
+* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
index, saving RAM in IndexReader; change default terms index
interval from 128 to 32, because the terms index now requires much
less RAM. (Robert Muir, Mike McCandless)
@@ -360,7 +375,7 @@ Bug fixes
Changes in backwards compatibility policy
* LUCENE-2719: Changed API of internal utility class
- org.apche.lucene.util.SorterTemplate to support faster quickSort using
+ org.apache.lucene.util.SorterTemplate to support faster quickSort using
pivot values and also merge sort and insertion sort. If you have used
this class, you have to implement two more methods for handling pivots.
(Uwe Schindler, Robert Muir, Mike McCandless)
@@ -632,7 +647,7 @@ New features
* LUCENE-2314: Added AttributeSource.copyTo(AttributeSource) that
allows to use cloneAttributes() and this method as a replacement
- for captureState()/restoreState(), if the state itsself
+ for captureState()/restoreState(), if the state itself
needs to be inspected/modified. (Uwe Schindler)
* LUCENE-2293: Expose control over max number of threads that
@@ -678,6 +693,12 @@ New features
Using this wrapper its easy to add fuzzy/wildcard to e.g. a SpanNearQuery.
(Robert Muir, Uwe Schindler)
+* LUCENE-2838: ConstantScoreQuery now directly supports wrapping a Query
+ instance for stripping off scores. The use of a QueryWrapperFilter
+ is no longer needed and discouraged for that use case. Directly wrapping
+ Query improves performance, as out-of-order collection is now supported.
+ (Uwe Schindler)
+
Optimizations
* LUCENE-2075: Terms dict cache is now shared across threads instead
@@ -689,7 +710,7 @@ Optimizations
* LUCENE-2136: If the multi reader (DirectoryReader or MultiReader)
only has a single sub-reader, delegate all enum requests to it.
- This avoid the overhead of using a PQ unecessarily. (Mike
+ This avoid the overhead of using a PQ unnecessarily. (Mike
McCandless)
* LUCENE-2137: Switch to AtomicInteger for some ref counting (Earwin
@@ -743,7 +764,7 @@ Optimizations
cases. (Mike McCandless)
* LUCENE-2719: Improved TermsHashPerField's sorting to use a better
- quick sort algorithm that dereferences the privot element not on
+ quick sort algorithm that dereferences the pivot element not on
every compare call. Also replaced lots of sorting code in Lucene
by the improved SorterTemplate class.
(Uwe Schindler, Robert Muir, Mike McCandless)
@@ -771,17 +792,21 @@ Build
* LUCENE-1709: Tests are now parallelized by default (except for benchmark). You
can force them to run sequentially by passing -Drunsequential=1 on the command
- line. The number of threads that are spwaned per CPU defaults to '1'. If you
+ line. The number of threads that are spawned per CPU defaults to '1'. If you
wish to change that, you can run the tests with -DthreadsPerProcessor=[num].
(Robert Muir, Shai Erera, Peter Kofler)
* LUCENE-2516: Backwards tests are now compiled against released lucene-core.jar
from tarball of previous version. Backwards tests are now packaged together
with src distribution. (Uwe Schindler)
+
+* LUCENE-2611: Added Ant target to install IntelliJ IDEA configuration:
+ "ant idea". See http://wiki.apache.org/lucene-java/HowtoConfigureIntelliJ
+ (Steven Rowe)
Test Cases
-* LUCENE-2037 Allow Junit4 tests in our envrionment (Erick Erickson
+* LUCENE-2037 Allow Junit4 tests in our environment (Erick Erickson
via Mike McCandless)
* LUCENE-1844: Speed up the unit tests (Mark Miller, Erick Erickson,
@@ -803,14 +828,14 @@ Test Cases
* LUCENE-2313, LUCENE-2322: Add VERBOSE to LuceneTestCase(J4) to control
verbosity of tests. If VERBOSE==false (default) tests should not print
anything other than errors to System.(out|err). The setting can be
- changed with -Dtests.verbose=true on test invokation.
+ changed with -Dtests.verbose=true on test invocation.
(Shai Erera, Paul Elschot, Uwe Schindler)
* LUCENE-2318: Remove inconsistent system property code for retrieving
temp and data directories inside test cases. It is now centralized in
LuceneTestCase(J4). Also changed lots of tests to use
getClass().getResourceAsStream() to retrieve test data. Tests needing
- access to "real" files from the test folder itsself, can use
+ access to "real" files from the test folder itself, can use
LuceneTestCase(J4).getDataFile(). (Uwe Schindler)
* LUCENE-2398, LUCENE-2611: Improve tests to work better from IDEs such
Modified: lucene/dev/branches/docvalues/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/build.xml?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/build.xml (original)
+++ lucene/dev/branches/docvalues/lucene/build.xml Wed Jan 5 20:25:17 2011
@@ -31,6 +31,7 @@
<path id="test.classpath">
<path refid="classpath"/>
<path refid="junit-path"/>
+ <path refid="ant-path"/>
<pathelement location="${build.dir}/classes/test"/>
</path>
@@ -44,7 +45,7 @@
<patternset id="src.dist.patterns"
includes="src/,build.xml,*build*.xml,docs/,*.txt,contrib/,*pom.xml*,lib/,backwards/"
- excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/,contrib/benchmark/temp/,contrib/benchmark/work/"
+ excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/"
/>
<patternset id="binary.build.dist.patterns"
includes="${final.name}.jar,docs/,contrib/*/*.jar,contrib/*/*.war, contrib/*/*/*.jar"
@@ -67,6 +68,7 @@
<path id="backwards.test.compile.classpath">
<path refid="junit-path"/>
+ <path refid="ant-path"/>
<fileset dir="${backwards.dir}/lib">
<include name="lucene-core*.jar"/>
</fileset>
@@ -206,7 +208,6 @@
<!-- Also remember to keep site.xml in sync. -->
<packageset dir="contrib/ant/src/java"/>
- <packageset dir="contrib/benchmark/src/java"/>
<packageset dir="contrib/db/bdb-je/src/java"/>
<packageset dir="contrib/db/bdb/src/java"/>
<packageset dir="contrib/demo/src/java"/>
@@ -232,7 +233,6 @@
<group title="Core" packages="org.apache.*:org.apache.lucene.analysis:org.apache.lucene.analysis.standard*:org.apache.lucene.analysis.tokenattributes*"/>
<group title="contrib: Ant" packages="org.apache.lucene.ant*"/>
- <group title="contrib: Benchmark" packages="org.apache.lucene.benchmark*"/>
<group title="contrib: Demo" packages="org.apache.lucene.demo*"/>
<group title="contrib: ICU" packages="org.apache.lucene.collation*"/>
<group title="contrib: DB" packages="org.apache.lucene.store.db*:org.apache.lucene.store.je*:com.sleepycat*"/>
Modified: lucene/dev/branches/docvalues/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/common-build.xml?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/common-build.xml (original)
+++ lucene/dev/branches/docvalues/lucene/common-build.xml Wed Jan 5 20:25:17 2011
@@ -53,6 +53,10 @@
<pathelement location="${junit-location.jar}"/>
</path>
+ <path id="ant-path">
+ <fileset dir="${common.dir}/lib" includes="ant-*.jar"/>
+ </path>
+
<!-- default arguments to pass to JVM executing tests -->
<property name="testmethod" value=""/>
<property name="args" value=""/>
@@ -71,7 +75,6 @@
<property name="tests.linedocsfile" value="europarl.lines.txt.gz" />
<property name="tests.iter" value="1" />
<property name="tests.seed" value="random" />
- <property name="tests.userdir" value="."/>
<property name="tests.loggingfile" value="/dev/null"/>
<property name="tests.nightly" value="false" />
@@ -81,6 +84,9 @@
<property name="javac.target" value="1.5"/>
<property name="javac.source.backwards" value="1.5"/>
<property name="javac.target.backwards" value="1.5"/>
+ <!-- clover wants to run with -lib, otherwise we prefer a repeatable
+ classpath -->
+ <property name="javac.includeAntRuntime" value="${run.clover}"/>
<property name="javadoc.link" value="http://java.sun.com/j2se/1.5/docs/api/"/>
<property name="javadoc.access" value="protected"/>
@@ -441,7 +447,7 @@
-->
<touch file="@{tempDir}/@{threadNum}/quiet.ant" verbose="false" mkdirs="true"/>
<junit printsummary="off" haltonfailure="no" maxmemory="512M" tempdir="@{tempDir}/@{threadNum}"
- errorProperty="tests.failed" failureProperty="tests.failed" forkmode="perBatch" dir="${tests.userdir}">
+ errorProperty="tests.failed" failureProperty="tests.failed" forkmode="perBatch" dir="@{tempDir}/@{threadNum}">
<classpath refid="@{junit.classpath}"/>
<assertions>
<enable package="org.apache.lucene"/>
@@ -655,6 +661,7 @@
<sequential>
<mkdir dir="@{destdir}"/>
<javac
+ includeAntRuntime="${javac.includeAntRuntime}"
encoding="${build.encoding}"
srcdir="@{srcdir}"
destdir="@{destdir}"
Modified: lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt Wed Jan 5 20:25:17 2011
@@ -7,6 +7,8 @@ Build
* LUCENE-2413: Moved the demo out of lucene core and into contrib/demo.
(Robert Muir)
+ * LUCENE-2845: Moved contrib/benchmark to modules.
+
New Features
* LUCENE-2604: Added RegexpQuery support to contrib/queryparser.
@@ -36,6 +38,9 @@ New Features
that doesn't synchronize on the file handle. This can be useful to
avoid the performance problems of SimpleFSDirectory and NIOFSDirectory.
(Robert Muir, Simon Willnauer, Uwe Schindler, Michael McCandless)
+
+ * LUCENE-2836: Add FieldCacheRewriteMethod, which rewrites MultiTermQueries
+ using the FieldCache's TermsEnum. (Robert Muir)
API Changes
@@ -73,6 +78,14 @@ Changes in backwards compatibility polic
* LUCENE-2581: Added new methods to FragmentsBuilder interface. These methods
are used to set pre/post tags and Encoder. (Koji Sekiguchi)
+
+ * LUCENE-2391: Improved spellchecker (re)build time/ram usage by omitting
+ frequencies/positions/norms for single-valued fields, modifying the default
+ ramBufferMBSize to match IndexWriterConfig (16MB), making index optimization
+ an optional boolean parameter, and modifying the incremental update logic
+ to work well with unoptimized spellcheck indexes. The indexDictionary() methods
+ were made final to ensure a hard backwards break in case you were subclassing
+ Spellchecker. In general, subclassing Spellchecker is not recommended. (Robert Muir)
Changes in runtime behavior
@@ -174,6 +187,9 @@ API Changes
* LUCENE-2747: Deprecated ArabicLetterTokenizer. StandardTokenizer now tokenizes
most languages correctly including Arabic. (Steven Rowe, Robert Muir)
+ * LUCENE-2830: Use StringBuilder instead of StringBuffer across Benchmark, and
+ remove the StringBuffer HtmlParser.parse() variant. (Shai Erera)
+
New features
* LUCENE-2306: Add NumericRangeFilter and NumericRangeQuery support to XMLQueryParser.
@@ -284,6 +300,8 @@ Build
* LUCENE-2797: Upgrade contrib/icu's ICU jar file to ICU 4.6
(Robert Muir)
+
+ * LUCENE-2833: Upgrade contrib/ant's jtidy jar file to r938 (Robert Muir)
Optimizations
Modified: lucene/dev/branches/docvalues/lucene/contrib/ant/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/ant/build.xml?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/ant/build.xml (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/ant/build.xml Wed Jan 5 20:25:17 2011
@@ -25,7 +25,7 @@
<path id="additional.dependencies">
<!-- TODO: make ${tidy.jar} property -->
- <pathelement location="lib/Tidy.jar"/>
+ <fileset dir="lib" includes="jtidy-*.jar"/>
</path>
<pathconvert property="project.classpath"
@@ -33,6 +33,8 @@
refid="additional.dependencies"
/>
+ <property name="javac.includeAntRuntime" value="true"/>
+
<import file="../contrib-build.xml"/>
<module-uptodate name="analysis/common" jarfile="${common.dir}/../modules/analysis/build/common/lucene-analyzers-common-${version}.jar"
Modified: lucene/dev/branches/docvalues/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java Wed Jan 5 20:25:17 2011
@@ -44,7 +44,6 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -268,7 +267,7 @@ public class IndexTask extends Task {
FSDirectory dir = FSDirectory.open(indexDir);
try {
- Searcher searcher = null;
+ IndexSearcher searcher = null;
boolean checkLastModified = false;
if (!create) {
try {
Modified: lucene/dev/branches/docvalues/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java Wed Jan 5 20:25:17 2011
@@ -18,14 +18,13 @@ package org.apache.lucene.ant;
*/
import java.io.File;
-import java.io.IOException;
+import java.io.IOException; // javadoc
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.store.Directory;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.types.FileSet;
@@ -39,7 +38,7 @@ public class IndexTaskTest extends Lucen
private final static String docHandler =
"org.apache.lucene.ant.FileExtensionDocumentHandler";
- private Searcher searcher;
+ private IndexSearcher searcher;
private Analyzer analyzer;
private Directory dir;
Modified: lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/FileDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/FileDocument.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/FileDocument.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/FileDocument.java Wed Jan 5 20:25:17 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.demo;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileReader;
import java.io.InputStreamReader;
import org.apache.lucene.document.DateTools;
Modified: lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java Wed Jan 5 20:25:17 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.demo;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;
@@ -28,7 +27,6 @@ import java.util.Date;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Collector;
@@ -36,7 +34,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -44,31 +41,12 @@ import org.apache.lucene.util.Version;
/** Simple command-line based search demo. */
public class SearchFiles {
- /** Use the norms from one field for all fields. Norms are read into memory,
- * using a byte of memory per document per searched field. This can cause
- * search of large collections with a large number of fields to run out of
- * memory. If all of the fields contain only a single token, then the norms
- * are all identical, then single norm vector may be shared. */
- private static class OneNormsReader extends FilterIndexReader {
- private String field;
-
- public OneNormsReader(IndexReader in, String field) {
- super(in);
- this.field = field;
- }
-
- @Override
- public byte[] norms(String field) throws IOException {
- return in.norms(this.field);
- }
- }
-
private SearchFiles() {}
/** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {
String usage =
- "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-raw] [-norms field] [-paging hitsPerPage]";
+ "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-raw] [-paging hitsPerPage]";
usage += "\n\tSpecify 'false' for hitsPerPage to use streaming instead of paging search.";
if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
System.out.println(usage);
@@ -80,7 +58,6 @@ public class SearchFiles {
String queries = null;
int repeat = 0;
boolean raw = false;
- String normsField = null;
boolean paging = true;
int hitsPerPage = 10;
@@ -99,9 +76,6 @@ public class SearchFiles {
i++;
} else if ("-raw".equals(args[i])) {
raw = true;
- } else if ("-norms".equals(args[i])) {
- normsField = args[i+1];
- i++;
} else if ("-paging".equals(args[i])) {
if (args[i+1].equals("false")) {
paging = false;
@@ -117,10 +91,7 @@ public class SearchFiles {
IndexReader reader = IndexReader.open(FSDirectory.open(new File(index)), true); // only searching, so read-only=true
- if (normsField != null)
- reader = new OneNormsReader(reader, normsField);
-
- Searcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = new IndexSearcher(reader);
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
BufferedReader in = null;
@@ -172,7 +143,7 @@ public class SearchFiles {
* This simulates the streaming search use case, where all hits are supposed to
* be processed, regardless of their relevance.
*/
- public static void doStreamingSearch(final Searcher searcher, Query query) throws IOException {
+ public static void doStreamingSearch(final IndexSearcher searcher, Query query) throws IOException {
Collector streamingHitCollector = new Collector() {
private Scorer scorer;
private int docBase;
@@ -214,7 +185,7 @@ public class SearchFiles {
* is executed another time and all hits are collected.
*
*/
- public static void doPagingSearch(BufferedReader in, Searcher searcher, Query query,
+ public static void doPagingSearch(BufferedReader in, IndexSearcher searcher, Query query,
int hitsPerPage, boolean raw, boolean interactive) throws IOException {
// Collect enough docs to show 5 pages
Modified: lucene/dev/branches/docvalues/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Wed Jan 5 20:25:17 2011
@@ -58,7 +58,6 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiPhraseQuery;
-import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PhraseQuery;
@@ -1301,68 +1300,6 @@ public class HighlighterTest extends Bas
assertEquals("XHTML Encoding should have worked:", rawDocContent, decodedSnippet);
}
- public void testMultiSearcher() throws Exception {
- // setup index 1
- Directory ramDir1 = newDirectory();
- IndexWriter writer1 = new IndexWriter(ramDir1, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
- Document d = new Document();
- Field f = new Field(FIELD_NAME, "multiOne", Field.Store.YES, Field.Index.ANALYZED);
- d.add(f);
- writer1.addDocument(d);
- writer1.optimize();
- writer1.close();
- IndexReader reader1 = IndexReader.open(ramDir1, true);
-
- // setup index 2
- Directory ramDir2 = newDirectory();
- IndexWriter writer2 = new IndexWriter(ramDir2, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
- d = new Document();
- f = new Field(FIELD_NAME, "multiTwo", Field.Store.YES, Field.Index.ANALYZED);
- d.add(f);
- writer2.addDocument(d);
- writer2.optimize();
- writer2.close();
- IndexReader reader2 = IndexReader.open(ramDir2, true);
-
- IndexSearcher searchers[] = new IndexSearcher[2];
- searchers[0] = new IndexSearcher(ramDir1, true);
- searchers[1] = new IndexSearcher(ramDir2, true);
- MultiSearcher multiSearcher = new MultiSearcher(searchers);
- QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
- parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
- query = parser.parse("multi*");
- if (VERBOSE) System.out.println("Searching for: " + query.toString(FIELD_NAME));
- // at this point the multisearcher calls combine(query[])
- hits = multiSearcher.search(query, null, 1000);
-
- // query = QueryParser.parse("multi*", FIELD_NAME, new StandardAnalyzer(TEST_VERSION));
- Query expandedQueries[] = new Query[2];
- expandedQueries[0] = query.rewrite(reader1);
- expandedQueries[1] = query.rewrite(reader2);
- query = query.combine(expandedQueries);
-
- // create an instance of the highlighter with the tags used to surround
- // highlighted text
- Highlighter highlighter = new Highlighter(this, new QueryTermScorer(query));
-
- for (int i = 0; i < hits.totalHits; i++) {
- String text = multiSearcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
- String highlightedText = highlighter.getBestFragment(tokenStream, text);
- if (VERBOSE) System.out.println(highlightedText);
- }
- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
- numHighlights == 2);
- reader1.close();
- reader2.close();
- searchers[0].close();
- searchers[1].close();
- ramDir1.close();
- ramDir2.close();
- }
-
public void testFieldSpecificHighlighting() throws Exception {
TestHighlightRunner helper = new TestHighlightRunner() {
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java Wed Jan 5 20:25:17 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.store.instanti
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import java.util.Arrays;
public class InstantiatedDocsAndPositionsEnum extends DocsAndPositionsEnum {
private int upto;
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Wed Jan 5 20:25:17 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Fields;
@@ -211,12 +212,11 @@ public class InstantiatedIndex
}
}
-
-
// create norms
for (String fieldName : allFieldNames) {
if (fields == null || fields.contains(fieldName)) {
- getNormsByFieldNameAndDocumentNumber().put(fieldName, sourceIndexReader.norms(fieldName));
+ byte norms[] = MultiNorms.norms(sourceIndexReader, fieldName);
+ getNormsByFieldNameAndDocumentNumber().put(fieldName, norms);
}
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Wed Jan 5 20:25:17 2011
@@ -39,7 +39,6 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.search.Similarity;
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java Wed Jan 5 20:25:17 2011
@@ -18,12 +18,11 @@ package org.apache.lucene.store.instanti
import java.io.IOException;
import java.util.Arrays;
-import java.util.Random;
-
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
@@ -69,8 +68,7 @@ public class TestEmptyIndex extends Luce
}
private void testNorms(IndexReader r) throws IOException {
- byte[] norms;
- norms = r.norms("foo");
+ byte[] norms = MultiNorms.norms(r, "foo");
if (norms != null) {
assertEquals(0, norms.length);
norms = new byte[10];
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Wed Jan 5 20:25:17 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
@@ -341,8 +342,8 @@ public class TestIndicesEquals extends L
// test norms as used by normal use
- byte[] aprioriNorms = aprioriReader.norms((String) field);
- byte[] testNorms = testReader.norms((String) field);
+ byte[] aprioriNorms = MultiNorms.norms(aprioriReader, (String) field);
+ byte[] testNorms = MultiNorms.norms(testReader, (String) field);
if (aprioriNorms != null) {
assertEquals(aprioriNorms.length, testNorms.length);
@@ -354,10 +355,10 @@ public class TestIndicesEquals extends L
// test norms as used by multireader
aprioriNorms = new byte[aprioriReader.maxDoc()];
- aprioriReader.norms((String) field, aprioriNorms, 0);
+ MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 0);
testNorms = new byte[testReader.maxDoc()];
- testReader.norms((String) field, testNorms, 0);
+ MultiNorms.norms(testReader, (String) field, testNorms, 0);
assertEquals(aprioriNorms.length, testNorms.length);
@@ -369,10 +370,10 @@ public class TestIndicesEquals extends L
// test norms as used by multireader
aprioriNorms = new byte[aprioriReader.maxDoc() + 10];
- aprioriReader.norms((String) field, aprioriNorms, 10);
+ MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 10);
testNorms = new byte[testReader.maxDoc() + 10];
- testReader.norms((String) field, testNorms, 10);
+ MultiNorms.norms(testReader, (String) field, testNorms, 10);
assertEquals(aprioriNorms.length, testNorms.length);
Modified: lucene/dev/branches/docvalues/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java Wed Jan 5 20:25:17 2011
@@ -53,11 +53,10 @@ import org.apache.lucene.queryParser.Mul
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.BytesRef;
@@ -73,7 +72,7 @@ class LuceneMethods {
private List<String> fields; //Fields as a vector
private List<String> indexedFields; //Fields as a vector
private String fieldsArray[]; //Fields as an array
- private Searcher searcher;
+ private IndexSearcher searcher;
private Query query; //current query string
private String analyzerClassFQN = null; // Analyzer class, if NULL, use default Analyzer
Modified: lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Wed Jan 5 20:25:17 2011
@@ -52,7 +52,6 @@ import org.apache.lucene.index.values.Do
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.RAMDirectory; // for javadocs
@@ -422,7 +421,7 @@ public class MemoryIndex implements Seri
if (query == null)
throw new IllegalArgumentException("query must not be null");
- Searcher searcher = createSearcher();
+ IndexSearcher searcher = createSearcher();
try {
final float[] scores = new float[1]; // inits to 0.0f (no match)
searcher.search(query, new Collector() {
@@ -739,7 +738,7 @@ public class MemoryIndex implements Seri
*/
private final class MemoryIndexReader extends IndexReader {
- private Searcher searcher; // needed to find searcher.getSimilarity()
+ private IndexSearcher searcher; // needed to find searcher.getSimilarity()
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
@@ -1148,7 +1147,7 @@ public class MemoryIndex implements Seri
return Similarity.getDefault();
}
- private void setSearcher(Searcher searcher) {
+ private void setSearcher(IndexSearcher searcher) {
this.searcher = searcher;
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java Wed Jan 5 20:25:17 2011
@@ -56,7 +56,7 @@ public class BalancedSegmentMergePolicy
@Override
protected long size(SegmentInfo info) throws IOException {
- long byteSize = info.sizeInBytes();
+ long byteSize = info.sizeInBytes(true);
float delRatio = (info.docCount <= 0 ? 0.0f : ((float)info.getDelCount() / (float)info.docCount));
return (info.docCount <= 0 ? byteSize : (long)((1.0f - delRatio) * byteSize));
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Wed Jan 5 20:25:17 2011
@@ -26,7 +26,6 @@ import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.FSDirectory;
@@ -108,7 +107,7 @@ public class IndexSplitter {
DecimalFormat formatter = new DecimalFormat("###,###.###");
for (int x = 0; x < infos.size(); x++) {
SegmentInfo info = infos.info(x);
- String sizeStr = formatter.format(info.sizeInBytes());
+ String sizeStr = formatter.format(info.sizeInBytes(true));
System.out.println(info.name + " " + sizeStr);
}
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Wed Jan 5 20:25:17 2011
@@ -84,7 +84,7 @@ public class TestFieldNormModifier exten
public void testFieldWithNoNorm() throws Exception {
IndexReader r = IndexReader.open(store, false);
- byte[] norms = r.norms("nonorm");
+ byte[] norms = MultiNorms.norms(r, "nonorm");
// sanity check, norms should all be 1
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -98,7 +98,7 @@ public class TestFieldNormModifier exten
// nothing should have changed
r = IndexReader.open(store, false);
- norms = r.norms("nonorm");
+ norms = MultiNorms.norms(r, "nonorm");
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
assertNull(norms);
@@ -183,14 +183,14 @@ public class TestFieldNormModifier exten
public void testNormKiller() throws IOException {
IndexReader r = IndexReader.open(store, false);
- byte[] oldNorms = r.norms("untokfield");
+ byte[] oldNorms = MultiNorms.norms(r, "untokfield");
r.close();
FieldNormModifier fnm = new FieldNormModifier(store, s);
fnm.reSetNorms("untokfield");
r = IndexReader.open(store, false);
- byte[] newNorms = r.norms("untokfield");
+ byte[] newNorms = MultiNorms.norms(r, "untokfield");
r.close();
assertFalse(Arrays.equals(oldNorms, newNorms));
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java Wed Jan 5 20:25:17 2011
@@ -17,8 +17,6 @@ package org.apache.lucene.misc;
* limitations under the License.
*/
-import java.util.Random;
-
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.BytesRef;
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Wed Jan 5 20:25:17 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldNormModifier;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DefaultSimilarity;
@@ -94,7 +95,7 @@ public class TestLengthNormModifier exte
public void testFieldWithNoNorm() throws Exception {
IndexReader r = IndexReader.open(store, false);
- byte[] norms = r.norms("nonorm");
+ byte[] norms = MultiNorms.norms(r, "nonorm");
// sanity check, norms should all be 1
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -112,7 +113,7 @@ public class TestLengthNormModifier exte
// nothing should have changed
r = IndexReader.open(store, false);
- norms = r.norms("nonorm");
+ norms = MultiNorms.norms(r, "nonorm");
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
assertNull(norms);
Modified: lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java Wed Jan 5 20:25:17 2011
@@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Similarity;
/**
* The BoostingQuery class can be used to effectively demote results that match a given query.
@@ -58,7 +58,7 @@ public class BoostingQuery extends Query
BooleanQuery result = new BooleanQuery() {
@Override
- public Similarity getSimilarity(Searcher searcher) {
+ public Similarity getSimilarity(IndexSearcher searcher) {
return new DefaultSimilarity() {
@Override
Modified: lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Wed Jan 5 20:25:17 2011
@@ -360,7 +360,7 @@ public class FuzzyLikeThisQuery extends
this.ignoreTF=ignoreTF;
}
@Override
- public Similarity getSimilarity(Searcher searcher)
+ public Similarity getSimilarity(IndexSearcher searcher)
{
Similarity result = super.getSimilarity(searcher);
result = new SimilarityDelegator(result) {
Modified: lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java Wed Jan 5 20:25:17 2011
@@ -29,11 +29,10 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryWrapperFilter;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.search.TopDocs;
@@ -195,7 +194,7 @@ public class ChainedFilterTest extends L
IndexReader reader = writer.getReader();
writer.close();
- Searcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = new IndexSearcher(reader);
Query query = new TermQuery(new Term("none", "none"));
Modified: lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Wed Jan 5 20:25:17 2011
@@ -18,8 +18,6 @@ package org.apache.lucene.search;
*/
import java.util.HashSet;
-import java.util.Random;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
Modified: lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java Wed Jan 5 20:25:17 2011
@@ -27,10 +27,8 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.spans.SpanFirstQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@@ -85,33 +83,6 @@ public class TestSpanRegexQuery extends
directory.close();
}
- public void testSpanRegexBug() throws CorruptIndexException, IOException {
- createRAMDirectories();
-
- SpanQuery srq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "a.*")));
- SpanQuery stq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "b.*")));
- SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { srq, stq }, 6,
- true);
-
- // 1. Search the same store which works
- IndexSearcher[] arrSearcher = new IndexSearcher[2];
- arrSearcher[0] = new IndexSearcher(indexStoreA, true);
- arrSearcher[1] = new IndexSearcher(indexStoreB, true);
- MultiSearcher searcher = new MultiSearcher(arrSearcher);
- int numHits = searcher.search(query, null, 1000).totalHits;
- arrSearcher[0].close();
- arrSearcher[1].close();
-
- // Will fail here
- // We expect 2 but only one matched
- // The rewriter function only write it once on the first IndexSearcher
- // So it's using term: a1 b1 to search on the second IndexSearcher
- // As a result, it won't match the document in the second IndexSearcher
- assertEquals(2, numHits);
- indexStoreA.close();
- indexStoreB.close();
- }
-
private void createRAMDirectories() throws CorruptIndexException,
LockObtainFailedException, IOException {
// creating a document to store
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java Wed Jan 5 20:25:17 2011
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.lucene.queryParser.core.QueryNodeException;
import org.apache.lucene.queryParser.core.config.QueryConfigHandler;
-import org.apache.lucene.queryParser.core.nodes.FieldQueryNode;
import org.apache.lucene.queryParser.core.nodes.FuzzyQueryNode;
import org.apache.lucene.queryParser.core.nodes.ParametricQueryNode;
import org.apache.lucene.queryParser.core.nodes.QueryNode;
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java Wed Jan 5 20:25:17 2011
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Query;
@@ -122,7 +121,7 @@ public class BooleanQueryTst {
/* if (verbose) System.out.println("Lucene: " + query.toString()); */
TestCollector tc = new TestCollector();
- Searcher searcher = new IndexSearcher(dBase.getDb(), true);
+ IndexSearcher searcher = new IndexSearcher(dBase.getDb(), true);
try {
searcher.search(query, tc);
} finally {
Modified: lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java Wed Jan 5 20:25:17 2011
@@ -17,8 +17,6 @@
package org.apache.lucene.spatial.geohash;
-import static junit.framework.Assert.*;
-
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
Modified: lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java Wed Jan 5 20:25:17 2011
@@ -1,7 +1,5 @@
package org.apache.lucene.spatial.geometry;
-import static junit.framework.Assert.*;
-
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
Modified: lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java Wed Jan 5 20:25:17 2011
@@ -252,7 +252,7 @@ public class TestCartesian extends Lucen
// Perform the search, using the term query, the serial chain filter, and the
// distance sort
- TopDocs hits = searcher.search(customScore.createWeight(searcher),null, 1000, sort);
+ TopDocs hits = searcher.search(customScore,null, 1000, sort);
int results = hits.totalHits;
ScoreDoc[] scoreDocs = hits.scoreDocs;
@@ -348,7 +348,7 @@ public class TestCartesian extends Lucen
// Perform the search, using the term query, the serial chain filter, and the
// distance sort
- TopDocs hits = searcher.search(customScore.createWeight(searcher),null, 1000, sort);
+ TopDocs hits = searcher.search(customScore,null, 1000, sort);
int results = hits.totalHits;
ScoreDoc[] scoreDocs = hits.scoreDocs;
@@ -444,7 +444,7 @@ public class TestCartesian extends Lucen
// Perform the search, using the term query, the serial chain filter, and the
// distance sort
- TopDocs hits = searcher.search(customScore.createWeight(searcher),null, 1000, sort);
+ TopDocs hits = searcher.search(customScore,null, 1000, sort);
int results = hits.totalHits;
ScoreDoc[] scoreDocs = hits.scoreDocs;
@@ -539,7 +539,7 @@ public class TestCartesian extends Lucen
// Perform the search, using the term query, the serial chain filter, and the
// distance sort
- TopDocs hits = searcher.search(customScore.createWeight(searcher),dq.getFilter(), 1000); //,sort);
+ TopDocs hits = searcher.search(customScore,dq.getFilter(), 1000); //,sort);
int results = hits.totalHits;
ScoreDoc[] scoreDocs = hits.scoreDocs;
Modified: lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Wed Jan 5 20:25:17 2011
@@ -18,8 +18,10 @@ package org.apache.lucene.search.spell;
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
@@ -30,6 +32,8 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
@@ -38,7 +42,10 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util.VirtualMethod;
/**
* <p>
@@ -492,35 +499,64 @@ public class SpellChecker implements jav
* @param dict Dictionary to index
* @param mergeFactor mergeFactor to use when indexing
* @param ramMB the max amount or memory in MB to use
+ * @param optimize whether or not the spellcheck index should be optimized
* @throws AlreadyClosedException if the Spellchecker is already closed
* @throws IOException
*/
- public void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
+ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException {
synchronized (modifyCurrentIndexLock) {
ensureOpen();
final Directory dir = this.spellIndex;
final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(mergeFactor);
+ IndexSearcher indexSearcher = obtainSearcher();
+ final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
+
+ if (searcher.maxDoc() > 0) {
+ new ReaderUtil.Gather(searcher.getIndexReader()) {
+ @Override
+ protected void add(int base, IndexReader r) throws IOException {
+ Terms terms = r.terms(F_WORD);
+ if (terms != null)
+ termsEnums.add(terms.iterator());
+ }
+ }.run();
+ }
+
+ boolean isEmpty = termsEnums.isEmpty();
+
+ try {
+ Iterator<String> iter = dict.getWordsIterator();
+ BytesRef currentTerm = new BytesRef();
+
+ terms: while (iter.hasNext()) {
+ String word = iter.next();
+
+ int len = word.length();
+ if (len < 3) {
+ continue; // too short we bail but "too long" is fine...
+ }
- Iterator<String> iter = dict.getWordsIterator();
- while (iter.hasNext()) {
- String word = iter.next();
-
- int len = word.length();
- if (len < 3) {
- continue; // too short we bail but "too long" is fine...
- }
+ if (!isEmpty) {
+ // we have a non-empty index, check if the term exists
+ currentTerm.copy(word);
+ for (TermsEnum te : termsEnums) {
+ if (te.seek(currentTerm, false) == TermsEnum.SeekStatus.FOUND) {
+ continue terms;
+ }
+ }
+ }
- if (this.exist(word)) { // if the word already exist in the gramindex
- continue;
+ // ok index the word
+ Document doc = createDocument(word, getMin(len), getMax(len));
+ writer.addDocument(doc);
}
-
- // ok index the word
- Document doc = createDocument(word, getMin(len), getMax(len));
- writer.addDocument(doc);
+ } finally {
+ releaseSearcher(indexSearcher);
}
// close writer
- writer.optimize();
+ if (optimize)
+ writer.optimize();
writer.close();
// also re-open the spell index to see our own changes when the next suggestion
// is fetched:
@@ -531,10 +567,21 @@ public class SpellChecker implements jav
/**
* Indexes the data from the given {@link Dictionary}.
* @param dict the dictionary to index
+ * @param mergeFactor mergeFactor to use when indexing
+ * @param ramMB the max amount or memory in MB to use
+ * @throws IOException
+ */
+ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
+ indexDictionary(dict, mergeFactor, ramMB, true);
+ }
+
+ /**
+ * Indexes the data from the given {@link Dictionary}.
+ * @param dict the dictionary to index
* @throws IOException
*/
- public void indexDictionary(Dictionary dict) throws IOException {
- indexDictionary(dict, 300, 10);
+ public final void indexDictionary(Dictionary dict) throws IOException {
+ indexDictionary(dict, 300, (int)IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
}
private static int getMin(int l) {
@@ -559,7 +606,12 @@ public class SpellChecker implements jav
private static Document createDocument(String text, int ng1, int ng2) {
Document doc = new Document();
- doc.add(new Field(F_WORD, text, Field.Store.YES, Field.Index.NOT_ANALYZED)); // orig term
+ // the word field is never queried on... its indexed so it can be quickly
+ // checked for rebuild (and stored for retrieval). Doesn't need norms or TF/pos
+ Field f = new Field(F_WORD, text, Field.Store.YES, Field.Index.NOT_ANALYZED);
+ f.setOmitTermFreqAndPositions(true);
+ f.setOmitNorms(true);
+ doc.add(f); // orig term
addGram(text, doc, ng1, ng2);
return doc;
}
@@ -573,12 +625,20 @@ public class SpellChecker implements jav
String gram = text.substring(i, i + ng);
doc.add(new Field(key, gram, Field.Store.NO, Field.Index.NOT_ANALYZED));
if (i == 0) {
- doc.add(new Field("start" + ng, gram, Field.Store.NO, Field.Index.NOT_ANALYZED));
+ // only one term possible in the startXXField, TF/pos and norms aren't needed.
+ Field startField = new Field("start" + ng, gram, Field.Store.NO, Field.Index.NOT_ANALYZED);
+ startField.setOmitTermFreqAndPositions(true);
+ startField.setOmitNorms(true);
+ doc.add(startField);
}
end = gram;
}
if (end != null) { // may not be present if len==ng1
- doc.add(new Field("end" + ng, end, Field.Store.NO, Field.Index.NOT_ANALYZED));
+ // only one term possible in the endXXField, TF/pos and norms aren't needed.
+ Field endField = new Field("end" + ng, end, Field.Store.NO, Field.Index.NOT_ANALYZED);
+ endField.setOmitTermFreqAndPositions(true);
+ endField.setOmitNorms(true);
+ doc.add(endField);
}
}
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SuggestWord.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SuggestWord.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SuggestWord.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SuggestWord.java Wed Jan 5 20:25:17 2011
@@ -1,7 +1,5 @@
package org.apache.lucene.search.spell;
-import java.util.Comparator;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Wed Jan 5 20:25:17 2011
@@ -39,7 +39,6 @@ import org.apache.lucene.search.Collecto
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -104,7 +103,7 @@ public final class SynExpand {
* @return the expanded Query
*/
public static Query expand( String query,
- Searcher syns,
+ IndexSearcher syns,
Analyzer a,
String f,
final float boost)
Modified: lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Wed Jan 5 20:25:17 2011
@@ -39,7 +39,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;
@@ -114,7 +113,7 @@ public class SynLookup {
* @param boost
*/
public static Query expand( String query,
- Searcher syns,
+ IndexSearcher syns,
Analyzer a,
final String field,
final float boost)
Modified: lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Wed Jan 5 20:25:17 2011
@@ -17,8 +17,6 @@ package org.apache.lucene.wordnet;
* limitations under the License.
*/
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.Reader;
@@ -27,8 +25,6 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
public class TestSynonymTokenFilter extends BaseTokenStreamTestCase {
final String testFile = "testSynonyms.txt";
Modified: lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java Wed Jan 5 20:25:17 2011
@@ -26,13 +26,12 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
public class TestWordnet extends LuceneTestCase {
- private Searcher searcher;
+ private IndexSearcher searcher;
private Directory dir;
String storePathName = new File(TEMP_DIR,"testLuceneWordnet").getAbsolutePath();
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java Wed Jan 5 20:25:17 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.document.NumericField; // for javadocs
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Document.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Document.java Wed Jan 5 20:25:17 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.document;
import java.util.*; // for javadoc
import org.apache.lucene.search.ScoreDoc; // for javadoc
-import org.apache.lucene.search.Searcher; // for javadoc
import org.apache.lucene.index.IndexReader; // for javadoc
/** Documents are the unit of indexing and search.
@@ -32,8 +31,7 @@ import org.apache.lucene.index.IndexRead
*
* <p>Note that fields which are <i>not</i> {@link Fieldable#isStored() stored} are
* <i>not</i> available in documents retrieved from the index, e.g. with {@link
- * ScoreDoc#doc}, {@link Searcher#doc(int)} or {@link
- * IndexReader#document(int)}.
+ * ScoreDoc#doc} or {@link IndexReader#document(int)}.
*/
public final class Document implements java.io.Serializable {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/document/Fieldable.java Wed Jan 5 20:25:17 2011
@@ -208,7 +208,7 @@ public interface Fieldable extends Seria
* required in the index, it also means any query
* requiring positional information, such as {@link
* PhraseQuery} or {@link SpanQuery} subclasses will
- * silently fail to find results.
+ * fail with an exception.
*/
void setOmitTermFreqAndPositions(boolean omitTermFreqAndPositions);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1055622&r1=1055621&r2=1055622&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java Wed Jan 5 20:25:17 2011
@@ -417,8 +417,8 @@ public class CheckIndex {
segInfoStat.hasProx = info.getHasProx();
msg(" numFiles=" + info.files().size());
segInfoStat.numFiles = info.files().size();
- msg(" size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
- segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
+ segInfoStat.sizeMB = info.sizeInBytes(true)/(1024.*1024.);
+ msg(" size (MB)=" + nf.format(segInfoStat.sizeMB));
Map<String,String> diagnostics = info.getDiagnostics();
segInfoStat.diagnostics = diagnostics;
if (diagnostics.size() > 0) {