You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/01/13 03:09:56 UTC
svn commit: r1058390 [2/16] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/
dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contr...
Modified: lucene/dev/branches/bulkpostings/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/build.xml?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/build.xml (original)
+++ lucene/dev/branches/bulkpostings/build.xml Thu Jan 13 02:09:33 2011
@@ -52,6 +52,101 @@
</sequential>
</target>
+ <target name="eclipse" description="Setup Eclipse configuration">
+ <copy file="dev-tools/eclipse/dot.classpath" tofile=".classpath" overwrite="true"/>
+ <mkdir dir=".settings"/>
+ <copy file="dev-tools/eclipse/resources.prefs"
+ tofile=".settings/org.eclipse.core.resources.prefs" overwrite="true"/>
+ <echo>Running Lucene contrib db/bdb-je task 'get-je-jar' ...</echo>
+ <subant target="get-je-jar">
+ <fileset dir="lucene/contrib/db/bdb-je" includes="build.xml" />
+ </subant>
+ <echo>Running Lucene contrib db/bdb task 'get-db-jar' ...</echo>
+ <subant target="get-db-jar">
+ <fileset dir="lucene/contrib/db/bdb" includes="build.xml" />
+ </subant>
+ <echo>
+ SUCCESS: You must right-click your project and choose Refresh
+ Please note, your project must use a Java 6 JRE
+ </echo>
+ </target>
+
+ <condition property="isMac"><os family="mac"/></condition>
+ <condition property="isUnix"><os family="unix"/></condition>
+ <condition property="isWindows"><os family="windows"/></condition>
+ <property environment="env"/>
+ <target name="idea" depends="copy-idea-files,finish-idea-setup-windows,
+ finish-idea-setup-mac,
+ finish-idea-setup-unix"
+ description="Setup IntelliJ IDEA configuration files"/>
+ <target name="copy-idea-files">
+ <copy todir=".">
+ <fileset dir="dev-tools/idea">
+ <exclude name="Intellij-Lucene-Codestyle.xml"/>
+ </fileset>
+ </copy>
+ <echo>Running Lucene contrib db/bdb-je task 'get-je-jar' ...</echo>
+ <subant target="get-je-jar">
+ <fileset dir="lucene/contrib/db/bdb-je" includes="build.xml" />
+ </subant>
+ <echo>Running Lucene contrib db/bdb task 'get-db-jar' ...</echo>
+ <subant target="get-db-jar">
+ <fileset dir="lucene/contrib/db/bdb" includes="build.xml" />
+ </subant>
+ </target>
+ <target name="finish-idea-setup-windows" if="isWindows">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools\idea\Intellij-Lucene-Codestyle.xml to
+ ${env.HOMEDRIVE}${env.HOMEPATH}\.IntelliJIdeaXX\config\codestyles\
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="finish-idea-setup-mac" if="isMac">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools/idea/Intellij-Lucene-Codestyle.xml to
+ ~/Library/Preferences/IntelliJXX/codestyles/
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="finish-idea-setup-unix" if="isUnix">
+ <echo>
+ To install the Lucene/Solr codestyle file, copy
+ dev-tools/idea/Intellij-Lucene-Codestyle.xml to
+ ~/.IntelliJIdeaXX/config/codestyles/
+ where "XX" is "90" for IntelliJ 9.0.X, "10" for 10.0.X, etc.
+ After restarting IntelliJ, select "Lucene"
+ from the dropdown list at:
+ Settings | Code Style | Use global settings | Scheme name
+
+ To complete IntelliJ IDEA setup, you must manually configure
+ Project Structure | Project | Project SDK.
+ </echo>
+ </target>
+ <target name="clean-idea"
+ description="Removes all IntelliJ IDEA configuration files">
+ <delete dir=".idea" failonerror="true"/>
+ <delete failonerror="true">
+ <fileset dir="." includes="*.iml,*.ipr,*.iws"/>
+ <fileset dir="solr" includes="**/*.iml"/>
+ <fileset dir="lucene" includes="**/*.iml"/>
+ <fileset dir="modules" includes="**/*.iml"/>
+ </delete>
+ </target>
+
<target name="clean" description="Clean Lucene and Solr">
<delete dir="dist" />
<sequential>
Modified: lucene/dev/branches/bulkpostings/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/CHANGES.txt?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/lucene/CHANGES.txt Thu Jan 13 02:09:33 2011
@@ -119,6 +119,22 @@ Changes in backwards compatibility polic
need to change it (e.g. using "\\" to escape '\' itself).
(Sunil Kamath, Terry Yang via Robert Muir)
+* LUCENE-2771: IndexReader.norms() now throws UOE on non-atomic IndexReaders. If
+ you really want a top-level norms, use MultiNorms or SlowMultiReaderWrapper.
+ (Uwe Schindler, Robert Muir)
+
+* LUCENE-2837: Collapsed Searcher, Searchable into IndexSearcher;
+ removed contrib/remote and MultiSearcher (Mike McCandless); absorbed
+ ParallelMultiSearcher into IndexSearcher as an optional
+ ExecutorServiced passed to its ctor. (Mike McCandless)
+
+* LUCENE-2837: Changed Weight#scorer, Weight#explain & Filter#getDocIdSet to
+ operate on a ReaderContext instead of directly on IndexReader to enable
+ searches to be aware of IndexSearcher's context. (Simon Willnauer)
+
+* LUCENE-2839: Scorer#score(Collector,int,int) is now public because it is
+ called from other classes and part of public API. (Uwe Schindler)
+
Changes in Runtime Behavior
* LUCENE-2650, LUCENE-2825: The behavior of FSDirectory.open has changed. On 64-bit
@@ -134,7 +150,7 @@ Changes in Runtime Behavior
* LUCENE-2805: IndexWriter now increments the index version on every change to
the index instead of for every commit. Committing or closing the IndexWriter
without any changes to the index will not cause any index version increment.
- (Simon Willnauer, Mike Mccandless)
+ (Simon Willnauer, Mike McCandless)
API Changes
@@ -298,7 +314,7 @@ New features
* LUCENE-995: The QueryParser now interprets * as an open end for range
queries. Literal asterisks may be represented by quoting or escaping
- (i.e. \* or "*") Custom QueryParser sublcasses overriding getRangeQuery()
+ (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery()
will be passed null for any open endpoint. (Adriano Crestani, yonik)
* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now
@@ -311,6 +327,12 @@ New features
extensions within the same segment. Codecs now use their per-segment codec
ID in the file names. (Simon Willnauer)
+* LUCENE-2843: Added a new terms index impl,
+ VariableGapTermsIndexWriter/Reader, that accepts a pluggable
+ IndexTermSelector for picking which terms should be indexed in the
+ terms dict. This impl stores the indexed terms in an FST, which is
+ much more RAM efficient than FixedGapTermsIndex. (Mike McCandless)
+
Optimizations
* LUCENE-2410: ~20% speedup on exact (slop=0) PhraseQuery matching.
@@ -324,7 +346,7 @@ Optimizations
efficient copying by sub-classes. Optimized copy is implemented for RAM and FS
streams. (Shai Erera)
-* LUCENE-2588: Don't store unecessary suffixes when writing the terms
+* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
index, saving RAM in IndexReader; change default terms index
interval from 128 to 32, because the terms index now requires much
less RAM. (Robert Muir, Mike McCandless)
@@ -360,7 +382,7 @@ Bug fixes
Changes in backwards compatibility policy
* LUCENE-2719: Changed API of internal utility class
- org.apche.lucene.util.SorterTemplate to support faster quickSort using
+ org.apache.lucene.util.SorterTemplate to support faster quickSort using
pivot values and also merge sort and insertion sort. If you have used
this class, you have to implement two more methods for handling pivots.
(Uwe Schindler, Robert Muir, Mike McCandless)
@@ -632,7 +654,7 @@ New features
* LUCENE-2314: Added AttributeSource.copyTo(AttributeSource) that
allows to use cloneAttributes() and this method as a replacement
- for captureState()/restoreState(), if the state itsself
+ for captureState()/restoreState(), if the state itself
needs to be inspected/modified. (Uwe Schindler)
* LUCENE-2293: Expose control over max number of threads that
@@ -678,6 +700,12 @@ New features
Using this wrapper its easy to add fuzzy/wildcard to e.g. a SpanNearQuery.
(Robert Muir, Uwe Schindler)
+* LUCENE-2838: ConstantScoreQuery now directly supports wrapping a Query
+ instance for stripping off scores. The use of a QueryWrapperFilter
+ is no longer needed and discouraged for that use case. Directly wrapping
+ Query improves performance, as out-of-order collection is now supported.
+ (Uwe Schindler)
+
Optimizations
* LUCENE-2075: Terms dict cache is now shared across threads instead
@@ -689,7 +717,7 @@ Optimizations
* LUCENE-2136: If the multi reader (DirectoryReader or MultiReader)
only has a single sub-reader, delegate all enum requests to it.
- This avoid the overhead of using a PQ unecessarily. (Mike
+ This avoid the overhead of using a PQ unnecessarily. (Mike
McCandless)
* LUCENE-2137: Switch to AtomicInteger for some ref counting (Earwin
@@ -743,7 +771,7 @@ Optimizations
cases. (Mike McCandless)
* LUCENE-2719: Improved TermsHashPerField's sorting to use a better
- quick sort algorithm that dereferences the privot element not on
+ quick sort algorithm that dereferences the pivot element not on
every compare call. Also replaced lots of sorting code in Lucene
by the improved SorterTemplate class.
(Uwe Schindler, Robert Muir, Mike McCandless)
@@ -771,17 +799,21 @@ Build
* LUCENE-1709: Tests are now parallelized by default (except for benchmark). You
can force them to run sequentially by passing -Drunsequential=1 on the command
- line. The number of threads that are spwaned per CPU defaults to '1'. If you
+ line. The number of threads that are spawned per CPU defaults to '1'. If you
wish to change that, you can run the tests with -DthreadsPerProcessor=[num].
(Robert Muir, Shai Erera, Peter Kofler)
* LUCENE-2516: Backwards tests are now compiled against released lucene-core.jar
from tarball of previous version. Backwards tests are now packaged together
with src distribution. (Uwe Schindler)
+
+* LUCENE-2611: Added Ant target to install IntelliJ IDEA configuration:
+ "ant idea". See http://wiki.apache.org/lucene-java/HowtoConfigureIntelliJ
+ (Steven Rowe)
Test Cases
-* LUCENE-2037 Allow Junit4 tests in our envrionment (Erick Erickson
+* LUCENE-2037 Allow Junit4 tests in our environment (Erick Erickson
via Mike McCandless)
* LUCENE-1844: Speed up the unit tests (Mark Miller, Erick Erickson,
@@ -803,14 +835,14 @@ Test Cases
* LUCENE-2313, LUCENE-2322: Add VERBOSE to LuceneTestCase(J4) to control
verbosity of tests. If VERBOSE==false (default) tests should not print
anything other than errors to System.(out|err). The setting can be
- changed with -Dtests.verbose=true on test invokation.
+ changed with -Dtests.verbose=true on test invocation.
(Shai Erera, Paul Elschot, Uwe Schindler)
* LUCENE-2318: Remove inconsistent system property code for retrieving
temp and data directories inside test cases. It is now centralized in
LuceneTestCase(J4). Also changed lots of tests to use
getClass().getResourceAsStream() to retrieve test data. Tests needing
- access to "real" files from the test folder itsself, can use
+ access to "real" files from the test folder itself, can use
LuceneTestCase(J4).getDataFile(). (Uwe Schindler)
* LUCENE-2398, LUCENE-2611: Improve tests to work better from IDEs such
Modified: lucene/dev/branches/bulkpostings/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/build.xml?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/build.xml (original)
+++ lucene/dev/branches/bulkpostings/lucene/build.xml Thu Jan 13 02:09:33 2011
@@ -31,6 +31,7 @@
<path id="test.classpath">
<path refid="classpath"/>
<path refid="junit-path"/>
+ <path refid="ant-path"/>
<pathelement location="${build.dir}/classes/test"/>
</path>
@@ -44,7 +45,7 @@
<patternset id="src.dist.patterns"
includes="src/,build.xml,*build*.xml,docs/,*.txt,contrib/,*pom.xml*,lib/,backwards/"
- excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/,contrib/benchmark/temp/,contrib/benchmark/work/"
+ excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/"
/>
<patternset id="binary.build.dist.patterns"
includes="${final.name}.jar,docs/,contrib/*/*.jar,contrib/*/*.war, contrib/*/*/*.jar"
@@ -67,6 +68,7 @@
<path id="backwards.test.compile.classpath">
<path refid="junit-path"/>
+ <path refid="ant-path"/>
<fileset dir="${backwards.dir}/lib">
<include name="lucene-core*.jar"/>
</fileset>
@@ -206,7 +208,6 @@
<!-- Also remember to keep site.xml in sync. -->
<packageset dir="contrib/ant/src/java"/>
- <packageset dir="contrib/benchmark/src/java"/>
<packageset dir="contrib/db/bdb-je/src/java"/>
<packageset dir="contrib/db/bdb/src/java"/>
<packageset dir="contrib/demo/src/java"/>
@@ -216,7 +217,6 @@
<packageset dir="contrib/memory/src/java"/>
<packageset dir="contrib/misc/src/java"/>
<packageset dir="contrib/queries/src/java"/>
- <packageset dir="contrib/remote/src/java"/>
<packageset dir="contrib/spatial/src/java"/>
<packageset dir="contrib/spellchecker/src/java"/>
<packageset dir="contrib/swing/src/java"/>
@@ -232,7 +232,6 @@
<group title="Core" packages="org.apache.*:org.apache.lucene.analysis:org.apache.lucene.analysis.standard*:org.apache.lucene.analysis.tokenattributes*"/>
<group title="contrib: Ant" packages="org.apache.lucene.ant*"/>
- <group title="contrib: Benchmark" packages="org.apache.lucene.benchmark*"/>
<group title="contrib: Demo" packages="org.apache.lucene.demo*"/>
<group title="contrib: ICU" packages="org.apache.lucene.collation*"/>
<group title="contrib: DB" packages="org.apache.lucene.store.db*:org.apache.lucene.store.je*:com.sleepycat*"/>
Modified: lucene/dev/branches/bulkpostings/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/common-build.xml?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/common-build.xml (original)
+++ lucene/dev/branches/bulkpostings/lucene/common-build.xml Thu Jan 13 02:09:33 2011
@@ -53,6 +53,10 @@
<pathelement location="${junit-location.jar}"/>
</path>
+ <path id="ant-path">
+ <fileset dir="${common.dir}/lib" includes="ant-*.jar"/>
+ </path>
+
<!-- default arguments to pass to JVM executing tests -->
<property name="testmethod" value=""/>
<property name="args" value=""/>
@@ -71,7 +75,6 @@
<property name="tests.linedocsfile" value="europarl.lines.txt.gz" />
<property name="tests.iter" value="1" />
<property name="tests.seed" value="random" />
- <property name="tests.userdir" value="."/>
<property name="tests.loggingfile" value="/dev/null"/>
<property name="tests.nightly" value="false" />
@@ -81,6 +84,9 @@
<property name="javac.target" value="1.5"/>
<property name="javac.source.backwards" value="1.5"/>
<property name="javac.target.backwards" value="1.5"/>
+ <!-- clover wants to run with -lib, otherwise we prefer a repeatable
+ classpath -->
+ <property name="javac.includeAntRuntime" value="${run.clover}"/>
<property name="javadoc.link" value="http://java.sun.com/j2se/1.5/docs/api/"/>
<property name="javadoc.access" value="protected"/>
@@ -441,7 +447,7 @@
-->
<touch file="@{tempDir}/@{threadNum}/quiet.ant" verbose="false" mkdirs="true"/>
<junit printsummary="off" haltonfailure="no" maxmemory="512M" tempdir="@{tempDir}/@{threadNum}"
- errorProperty="tests.failed" failureProperty="tests.failed" forkmode="perBatch" dir="${tests.userdir}">
+ errorProperty="tests.failed" failureProperty="tests.failed" forkmode="perBatch" dir="@{tempDir}/@{threadNum}">
<classpath refid="@{junit.classpath}"/>
<assertions>
<enable package="org.apache.lucene"/>
@@ -655,6 +661,7 @@
<sequential>
<mkdir dir="@{destdir}"/>
<javac
+ includeAntRuntime="${javac.includeAntRuntime}"
encoding="${build.encoding}"
srcdir="@{srcdir}"
destdir="@{destdir}"
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/CHANGES.txt?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/CHANGES.txt Thu Jan 13 02:09:33 2011
@@ -7,6 +7,8 @@ Build
* LUCENE-2413: Moved the demo out of lucene core and into contrib/demo.
(Robert Muir)
+ * LUCENE-2845: Moved contrib/benchmark to modules.
+
New Features
* LUCENE-2604: Added RegexpQuery support to contrib/queryparser.
@@ -36,6 +38,9 @@ New Features
that doesn't synchronize on the file handle. This can be useful to
avoid the performance problems of SimpleFSDirectory and NIOFSDirectory.
(Robert Muir, Simon Willnauer, Uwe Schindler, Michael McCandless)
+
+ * LUCENE-2836: Add FieldCacheRewriteMethod, which rewrites MultiTermQueries
+ using the FieldCache's TermsEnum. (Robert Muir)
API Changes
@@ -73,6 +78,14 @@ Changes in backwards compatibility polic
* LUCENE-2581: Added new methods to FragmentsBuilder interface. These methods
are used to set pre/post tags and Encoder. (Koji Sekiguchi)
+
+ * LUCENE-2391: Improved spellchecker (re)build time/ram usage by omitting
+ frequencies/positions/norms for single-valued fields, modifying the default
+ ramBufferMBSize to match IndexWriterConfig (16MB), making index optimization
+ an optional boolean parameter, and modifying the incremental update logic
+ to work well with unoptimized spellcheck indexes. The indexDictionary() methods
+ were made final to ensure a hard backwards break in case you were subclassing
+ Spellchecker. In general, subclassing Spellchecker is not recommended. (Robert Muir)
Changes in runtime behavior
@@ -268,6 +281,9 @@ New features
BooleanModifiersQueryNodeProcessor, for example instead of GroupQueryNodeProcessor.
(Adriano Crestani via Robert Muir)
+ * LUCENE-2842: Add analyzer for Galician. Also adds the RSLP (Orengo) stemmer
+ for Portuguese. (Robert Muir)
+
Build
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/ant/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/ant/build.xml?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/ant/build.xml (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/ant/build.xml Thu Jan 13 02:09:33 2011
@@ -33,6 +33,8 @@
refid="additional.dependencies"
/>
+ <property name="javac.includeAntRuntime" value="true"/>
+
<import file="../contrib-build.xml"/>
<module-uptodate name="analysis/common" jarfile="${common.dir}/../modules/analysis/build/common/lucene-analyzers-common-${version}.jar"
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java Thu Jan 13 02:09:33 2011
@@ -44,7 +44,6 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -268,7 +267,7 @@ public class IndexTask extends Task {
FSDirectory dir = FSDirectory.open(indexDir);
try {
- Searcher searcher = null;
+ IndexSearcher searcher = null;
boolean checkLastModified = false;
if (!create) {
try {
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java Thu Jan 13 02:09:33 2011
@@ -18,14 +18,13 @@ package org.apache.lucene.ant;
*/
import java.io.File;
-import java.io.IOException;
+import java.io.IOException; // javadoc
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.store.Directory;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.types.FileSet;
@@ -39,7 +38,7 @@ public class IndexTaskTest extends Lucen
private final static String docHandler =
"org.apache.lucene.ant.FileExtensionDocumentHandler";
- private Searcher searcher;
+ private IndexSearcher searcher;
private Analyzer analyzer;
private Directory dir;
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/demo/src/java/org/apache/lucene/demo/SearchFiles.java Thu Jan 13 02:09:33 2011
@@ -27,7 +27,6 @@ import java.util.Date;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Collector;
@@ -35,7 +34,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -43,31 +41,12 @@ import org.apache.lucene.util.Version;
/** Simple command-line based search demo. */
public class SearchFiles {
- /** Use the norms from one field for all fields. Norms are read into memory,
- * using a byte of memory per document per searched field. This can cause
- * search of large collections with a large number of fields to run out of
- * memory. If all of the fields contain only a single token, then the norms
- * are all identical, then single norm vector may be shared. */
- private static class OneNormsReader extends FilterIndexReader {
- private String field;
-
- public OneNormsReader(IndexReader in, String field) {
- super(in);
- this.field = field;
- }
-
- @Override
- public byte[] norms(String field) throws IOException {
- return in.norms(this.field);
- }
- }
-
private SearchFiles() {}
/** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {
String usage =
- "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-raw] [-norms field] [-paging hitsPerPage]";
+ "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-raw] [-paging hitsPerPage]";
usage += "\n\tSpecify 'false' for hitsPerPage to use streaming instead of paging search.";
if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
System.out.println(usage);
@@ -79,7 +58,6 @@ public class SearchFiles {
String queries = null;
int repeat = 0;
boolean raw = false;
- String normsField = null;
boolean paging = true;
int hitsPerPage = 10;
@@ -98,9 +76,6 @@ public class SearchFiles {
i++;
} else if ("-raw".equals(args[i])) {
raw = true;
- } else if ("-norms".equals(args[i])) {
- normsField = args[i+1];
- i++;
} else if ("-paging".equals(args[i])) {
if (args[i+1].equals("false")) {
paging = false;
@@ -116,10 +91,7 @@ public class SearchFiles {
IndexReader reader = IndexReader.open(FSDirectory.open(new File(index)), true); // only searching, so read-only=true
- if (normsField != null)
- reader = new OneNormsReader(reader, normsField);
-
- Searcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = new IndexSearcher(reader);
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
BufferedReader in = null;
@@ -171,7 +143,7 @@ public class SearchFiles {
* This simulates the streaming search use case, where all hits are supposed to
* be processed, regardless of their relevance.
*/
- public static void doStreamingSearch(final Searcher searcher, Query query) throws IOException {
+ public static void doStreamingSearch(final IndexSearcher searcher, Query query) throws IOException {
Collector streamingHitCollector = new Collector() {
private Scorer scorer;
private int docBase;
@@ -213,7 +185,7 @@ public class SearchFiles {
* is executed another time and all hits are collected.
*
*/
- public static void doPagingSearch(BufferedReader in, Searcher searcher, Query query,
+ public static void doPagingSearch(BufferedReader in, IndexSearcher searcher, Query query,
int hitsPerPage, boolean raw, boolean interactive) throws IOException {
// Collect enough docs to show 5 pages
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Thu Jan 13 02:09:33 2011
@@ -58,7 +58,6 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiPhraseQuery;
-import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PhraseQuery;
@@ -1301,68 +1300,6 @@ public class HighlighterTest extends Bas
assertEquals("XHTML Encoding should have worked:", rawDocContent, decodedSnippet);
}
- public void testMultiSearcher() throws Exception {
- // setup index 1
- Directory ramDir1 = newDirectory();
- IndexWriter writer1 = new IndexWriter(ramDir1, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
- Document d = new Document();
- Field f = new Field(FIELD_NAME, "multiOne", Field.Store.YES, Field.Index.ANALYZED);
- d.add(f);
- writer1.addDocument(d);
- writer1.optimize();
- writer1.close();
- IndexReader reader1 = IndexReader.open(ramDir1, true);
-
- // setup index 2
- Directory ramDir2 = newDirectory();
- IndexWriter writer2 = new IndexWriter(ramDir2, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
- d = new Document();
- f = new Field(FIELD_NAME, "multiTwo", Field.Store.YES, Field.Index.ANALYZED);
- d.add(f);
- writer2.addDocument(d);
- writer2.optimize();
- writer2.close();
- IndexReader reader2 = IndexReader.open(ramDir2, true);
-
- IndexSearcher searchers[] = new IndexSearcher[2];
- searchers[0] = new IndexSearcher(ramDir1, true);
- searchers[1] = new IndexSearcher(ramDir2, true);
- MultiSearcher multiSearcher = new MultiSearcher(searchers);
- QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
- parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
- query = parser.parse("multi*");
- if (VERBOSE) System.out.println("Searching for: " + query.toString(FIELD_NAME));
- // at this point the multisearcher calls combine(query[])
- hits = multiSearcher.search(query, null, 1000);
-
- // query = QueryParser.parse("multi*", FIELD_NAME, new StandardAnalyzer(TEST_VERSION));
- Query expandedQueries[] = new Query[2];
- expandedQueries[0] = query.rewrite(reader1);
- expandedQueries[1] = query.rewrite(reader2);
- query = query.combine(expandedQueries);
-
- // create an instance of the highlighter with the tags used to surround
- // highlighted text
- Highlighter highlighter = new Highlighter(this, new QueryTermScorer(query));
-
- for (int i = 0; i < hits.totalHits; i++) {
- String text = multiSearcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
- String highlightedText = highlighter.getBestFragment(tokenStream, text);
- if (VERBOSE) System.out.println(highlightedText);
- }
- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
- numHighlights == 2);
- reader1.close();
- reader2.close();
- searchers[0].close();
- searchers[1].close();
- ramDir1.close();
- ramDir2.close();
- }
-
public void testFieldSpecificHighlighting() throws Exception {
TestHighlightRunner helper = new TestHighlightRunner() {
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Thu Jan 13 02:09:33 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Fields;
@@ -211,12 +212,11 @@ public class InstantiatedIndex
}
}
-
-
// create norms
for (String fieldName : allFieldNames) {
if (fields == null || fields.contains(fieldName)) {
- getNormsByFieldNameAndDocumentNumber().put(fieldName, sourceIndexReader.norms(fieldName));
+ byte norms[] = MultiNorms.norms(sourceIndexReader, fieldName);
+ getNormsByFieldNameAndDocumentNumber().put(fieldName, norms);
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Thu Jan 13 02:09:33 2011
@@ -31,6 +31,7 @@ import java.util.Comparator;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.*;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.BytesRef;
@@ -45,6 +46,8 @@ import org.apache.lucene.util.Bits;
public class InstantiatedIndexReader extends IndexReader {
private final InstantiatedIndex index;
+ private ReaderContext context = new AtomicReaderContext(this);
+
public InstantiatedIndexReader(InstantiatedIndex index) {
super();
@@ -424,6 +427,11 @@ public class InstantiatedIndexReader ext
}
};
}
+
+ @Override
+ public ReaderContext getTopReaderContext() {
+ return context;
+ }
@Override
public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Thu Jan 13 02:09:33 2011
@@ -38,6 +38,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermVectorOffsetInfo;
@@ -235,9 +236,10 @@ public class InstantiatedIndexWriter imp
termsInDocument += eFieldTermDocInfoFactoriesByTermText.getValue().size();
if (eFieldTermDocInfoFactoriesByTermText.getKey().indexed && !eFieldTermDocInfoFactoriesByTermText.getKey().omitNorms) {
- float norm = eFieldTermDocInfoFactoriesByTermText.getKey().boost;
- norm *= document.getDocument().getBoost();
- norm *= similarity.lengthNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
+ final FieldInvertState invertState = new FieldInvertState();
+ invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
+ invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
+ final float norm = similarity.computeNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, invertState);
normsByFieldNameAndDocumentNumber.get(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName)[document.getDocumentNumber()] = similarity.encodeNormValue(norm);
} else {
System.currentTimeMillis();
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java Thu Jan 13 02:09:33 2011
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
@@ -67,8 +68,7 @@ public class TestEmptyIndex extends Luce
}
private void testNorms(IndexReader r) throws IOException {
- byte[] norms;
- norms = r.norms("foo");
+ byte[] norms = MultiNorms.norms(r, "foo");
if (norms != null) {
assertEquals(0, norms.length);
norms = new byte[10];
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Thu Jan 13 02:09:33 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
@@ -341,8 +342,8 @@ public class TestIndicesEquals extends L
// test norms as used by normal use
- byte[] aprioriNorms = aprioriReader.norms((String) field);
- byte[] testNorms = testReader.norms((String) field);
+ byte[] aprioriNorms = MultiNorms.norms(aprioriReader, (String) field);
+ byte[] testNorms = MultiNorms.norms(testReader, (String) field);
if (aprioriNorms != null) {
assertEquals(aprioriNorms.length, testNorms.length);
@@ -354,10 +355,10 @@ public class TestIndicesEquals extends L
// test norms as used by multireader
aprioriNorms = new byte[aprioriReader.maxDoc()];
- aprioriReader.norms((String) field, aprioriNorms, 0);
+ MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 0);
testNorms = new byte[testReader.maxDoc()];
- testReader.norms((String) field, testNorms, 0);
+ MultiNorms.norms(testReader, (String) field, testNorms, 0);
assertEquals(aprioriNorms.length, testNorms.length);
@@ -369,10 +370,10 @@ public class TestIndicesEquals extends L
// test norms as used by multireader
aprioriNorms = new byte[aprioriReader.maxDoc() + 10];
- aprioriReader.norms((String) field, aprioriNorms, 10);
+ MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 10);
testNorms = new byte[testReader.maxDoc() + 10];
- testReader.norms((String) field, testNorms, 10);
+ MultiNorms.norms(testReader, (String) field, testNorms, 10);
assertEquals(aprioriNorms.length, testNorms.length);
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java Thu Jan 13 02:09:33 2011
@@ -53,11 +53,10 @@ import org.apache.lucene.queryParser.Mul
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.BytesRef;
@@ -73,7 +72,7 @@ class LuceneMethods {
private List<String> fields; //Fields as a vector
private List<String> indexedFields; //Fields as a vector
private String fieldsArray[]; //Fields as an array
- private Searcher searcher;
+ private IndexSearcher searcher;
private Query query; //current query string
private String analyzerClassFQN = null; // Analyzer class, if NULL, use default Analyzer
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Jan 13 02:09:33 2011
@@ -49,10 +49,10 @@ import org.apache.lucene.index.TermFreqV
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorMapper;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.RAMDirectory; // for javadocs
@@ -422,7 +422,7 @@ public class MemoryIndex implements Seri
if (query == null)
throw new IllegalArgumentException("query must not be null");
- Searcher searcher = createSearcher();
+ IndexSearcher searcher = createSearcher();
try {
final float[] scores = new float[1]; // inits to 0.0f (no match)
searcher.search(query, new Collector() {
@@ -739,7 +739,8 @@ public class MemoryIndex implements Seri
*/
private final class MemoryIndexReader extends IndexReader {
- private Searcher searcher; // needed to find searcher.getSimilarity()
+ private IndexSearcher searcher; // needed to find searcher.getSimilarity()
+ private final ReaderContext readerInfos = new AtomicReaderContext(this);
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
@@ -766,6 +767,11 @@ public class MemoryIndex implements Seri
if (DEBUG) System.err.println("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
return freq;
}
+
+ @Override
+ public ReaderContext getTopReaderContext() {
+ return readerInfos;
+ }
@Override
public Fields fields() {
@@ -1314,7 +1320,7 @@ public class MemoryIndex implements Seri
return Similarity.getDefault();
}
- private void setSearcher(Searcher searcher) {
+ private void setSearcher(IndexSearcher searcher) {
this.searcher = searcher;
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Thu Jan 13 02:09:33 2011
@@ -33,7 +33,7 @@ import org.apache.lucene.util.ReaderUtil
/**
* Given a directory and a list of fields, updates the fieldNorms in place for every document.
*
- * If Similarity class is specified, uses its lengthNorm method to set norms.
+ * If Similarity class is specified, uses its computeNorm method to set norms.
* If -n command line argument is used, removed field norms, as if
* {@link org.apache.lucene.document.Field.Index}.NO_NORMS was used.
*
@@ -119,6 +119,7 @@ public class FieldNormModifier {
final List<IndexReader> subReaders = new ArrayList<IndexReader>();
ReaderUtil.gatherSubReaders(subReaders, reader);
+ final FieldInvertState invertState = new FieldInvertState();
for(IndexReader subReader : subReaders) {
final Bits delDocs = subReader.getDeletedDocs();
@@ -143,9 +144,11 @@ public class FieldNormModifier {
}
}
+ invertState.setBoost(1.0f);
for (int d = 0; d < termCounts.length; d++) {
if (delDocs == null || !delDocs.get(d)) {
- subReader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+ invertState.setLength(termCounts[d]);
+ subReader.setNorm(d, fieldName, sim.encodeNormValue(sim.computeNorm(fieldName, invertState)));
}
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Thu Jan 13 02:09:33 2011
@@ -146,7 +146,7 @@ public class SweetSpotSimilarity extends
else
numTokens = state.getLength();
- return state.getBoost() * lengthNorm(fieldName, numTokens);
+ return state.getBoost() * computeLengthNorm(fieldName, numTokens);
}
/**
@@ -167,8 +167,7 @@ public class SweetSpotSimilarity extends
*
* @see #setLengthNormFactors
*/
- @Override
- public float lengthNorm(String fieldName, int numTerms) {
+ public float computeLengthNorm(String fieldName, int numTerms) {
int l = ln_min;
int h = ln_max;
float s = ln_steep;
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Thu Jan 13 02:09:33 2011
@@ -43,8 +43,8 @@ public class TestFieldNormModifier exten
/** inverts the normal notion of lengthNorm */
public static Similarity s = new DefaultSimilarity() {
@Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
}
};
@@ -84,7 +84,7 @@ public class TestFieldNormModifier exten
public void testFieldWithNoNorm() throws Exception {
IndexReader r = IndexReader.open(store, false);
- byte[] norms = r.norms("nonorm");
+ byte[] norms = MultiNorms.norms(r, "nonorm");
// sanity check, norms should all be 1
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -98,7 +98,7 @@ public class TestFieldNormModifier exten
// nothing should have changed
r = IndexReader.open(store, false);
- norms = r.norms("nonorm");
+ norms = MultiNorms.norms(r, "nonorm");
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
assertNull(norms);
@@ -183,14 +183,14 @@ public class TestFieldNormModifier exten
public void testNormKiller() throws IOException {
IndexReader r = IndexReader.open(store, false);
- byte[] oldNorms = r.norms("untokfield");
+ byte[] oldNorms = MultiNorms.norms(r, "untokfield");
r.close();
FieldNormModifier fnm = new FieldNormModifier(store, s);
fnm.reSetNorms("untokfield");
r = IndexReader.open(store, false);
- byte[] newNorms = r.norms("untokfield");
+ byte[] newNorms = MultiNorms.norms(r, "untokfield");
r.close();
assertFalse(Arrays.equals(oldNorms, newNorms));
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Thu Jan 13 02:09:33 2011
@@ -21,13 +21,14 @@ package org.apache.lucene.misc;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.FieldInvertState;
/**
* Test of the SweetSpotSimilarity
*/
public class SweetSpotSimilarityTest extends LuceneTestCase {
- public void testSweetSpotLengthNorm() {
+ public void testSweetSpotComputeNorm() {
SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.setLengthNormFactors(1,1,0.5f);
@@ -37,10 +38,13 @@ public class SweetSpotSimilarityTest ext
// base case, should degrade
-
+ final FieldInvertState invertState = new FieldInvertState();
+ invertState.setBoost(1.0f);
for (int i = 1; i < 1000; i++) {
+ invertState.setLength(i);
assertEquals("base case: i="+i,
- d.lengthNorm("foo",i), s.lengthNorm("foo",i),
+ d.computeNorm("foo", invertState),
+ s.computeNorm("foo", invertState),
0.0f);
}
@@ -49,14 +53,21 @@ public class SweetSpotSimilarityTest ext
ss.setLengthNormFactors(3,10,0.5f);
for (int i = 3; i <=10; i++) {
+ invertState.setLength(i);
assertEquals("3,10: spot i="+i,
- 1.0f, s.lengthNorm("foo",i),
+ 1.0f,
+ s.computeNorm("foo", invertState),
0.0f);
}
for (int i = 10; i < 1000; i++) {
+ invertState.setLength(i-9);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("foo", invertState);
assertEquals("3,10: 10<x : i="+i,
- d.lengthNorm("foo",i-9), s.lengthNorm("foo",i),
+ normD,
+ normS,
0.0f);
}
@@ -68,33 +79,54 @@ public class SweetSpotSimilarityTest ext
for (int i = 3; i <=10; i++) {
+ invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
- 1.0f, s.lengthNorm("foo",i),
+ 1.0f,
+ s.computeNorm("foo", invertState),
0.0f);
}
for (int i = 10; i < 1000; i++) {
+ invertState.setLength(i-9);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("foo", invertState);
assertEquals("f: 3,10: 10<x : i="+i,
- d.lengthNorm("foo",i-9), s.lengthNorm("foo",i),
+ normD,
+ normS,
0.0f);
}
for (int i = 8; i <=13; i++) {
+ invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
- 1.0f, s.lengthNorm("bar",i),
+ 1.0f,
+ s.computeNorm("bar", invertState),
0.0f);
}
for (int i = 6; i <=9; i++) {
+ invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
- 1.0f, s.lengthNorm("yak",i),
+ 1.0f,
+ s.computeNorm("yak", invertState),
0.0f);
}
for (int i = 13; i < 1000; i++) {
+ invertState.setLength(i-12);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("bar", invertState);
assertEquals("f: 8,13: 13<x : i="+i,
- d.lengthNorm("foo",i-12), s.lengthNorm("bar",i),
+ normD,
+ normS,
0.0f);
}
for (int i = 9; i < 1000; i++) {
+ invertState.setLength(i-8);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("yak", invertState);
assertEquals("f: 6,9: 9<x : i="+i,
- d.lengthNorm("foo",i-8), s.lengthNorm("yak",i),
+ normD,
+ normS,
0.0f);
}
@@ -105,9 +137,12 @@ public class SweetSpotSimilarityTest ext
ss.setLengthNormFactors("b",5,8,0.1f, false);
for (int i = 9; i < 1000; i++) {
- assertTrue("s: i="+i+" : a="+ss.lengthNorm("a",i)+
- " < b="+ss.lengthNorm("b",i),
- ss.lengthNorm("a",i) < s.lengthNorm("b",i));
+ invertState.setLength(i);
+ final float normSS = ss.computeNorm("a", invertState);
+ final float normS = s.computeNorm("b", invertState);
+ assertTrue("s: i="+i+" : a="+normSS+
+ " < b="+normS,
+ normSS < normS);
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Thu Jan 13 02:09:33 2011
@@ -22,9 +22,11 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.FieldNormModifier;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiNorms;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DefaultSimilarity;
@@ -45,11 +47,11 @@ public class TestLengthNormModifier exte
/** inverts the normal notion of lengthNorm */
public static Similarity s = new DefaultSimilarity() {
- @Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
- }
- };
+ @Override
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ }
+ };
@Override
public void setUp() throws Exception {
@@ -94,7 +96,7 @@ public class TestLengthNormModifier exte
public void testFieldWithNoNorm() throws Exception {
IndexReader r = IndexReader.open(store, false);
- byte[] norms = r.norms("nonorm");
+ byte[] norms = MultiNorms.norms(r, "nonorm");
// sanity check, norms should all be 1
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
@@ -112,7 +114,7 @@ public class TestLengthNormModifier exte
// nothing should have changed
r = IndexReader.open(store, false);
- norms = r.norms("nonorm");
+ norms = MultiNorms.norms(r, "nonorm");
assertTrue("Whoops we have norms?", !r.hasNorms("nonorm"));
assertNull(norms);
@@ -161,11 +163,11 @@ public class TestLengthNormModifier exte
// override the norms to be inverted
Similarity s = new DefaultSimilarity() {
- @Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
- }
- };
+ @Override
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ }
+ };
FieldNormModifier fnm = new FieldNormModifier(store, s);
fnm.reSetNorms("field");
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java Thu Jan 13 02:09:33 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetDISI;
@@ -41,10 +42,10 @@ public class BooleanFilter extends Filte
ArrayList<Filter> notFilters = null;
ArrayList<Filter> mustFilters = null;
- private DocIdSetIterator getDISI(ArrayList<Filter> filters, int index, IndexReader reader)
+ private DocIdSetIterator getDISI(ArrayList<Filter> filters, int index, AtomicReaderContext context)
throws IOException
{
- return filters.get(index).getDocIdSet(reader).iterator();
+ return filters.get(index).getDocIdSet(context).iterator();
}
/**
@@ -52,21 +53,21 @@ public class BooleanFilter extends Filte
* of the filters that have been added.
*/
@Override
- public DocIdSet getDocIdSet(IndexReader reader) throws IOException
+ public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException
{
OpenBitSetDISI res = null;
-
+ final IndexReader reader = context.reader;
if (shouldFilters != null) {
for (int i = 0; i < shouldFilters.size(); i++) {
if (res == null) {
- res = new OpenBitSetDISI(getDISI(shouldFilters, i, reader), reader.maxDoc());
+ res = new OpenBitSetDISI(getDISI(shouldFilters, i, context), reader.maxDoc());
} else {
- DocIdSet dis = shouldFilters.get(i).getDocIdSet(reader);
+ DocIdSet dis = shouldFilters.get(i).getDocIdSet(context);
if(dis instanceof OpenBitSet) {
// optimized case for OpenBitSets
res.or((OpenBitSet) dis);
} else {
- res.inPlaceOr(getDISI(shouldFilters, i, reader));
+ res.inPlaceOr(getDISI(shouldFilters, i, context));
}
}
}
@@ -75,15 +76,15 @@ public class BooleanFilter extends Filte
if (notFilters!=null) {
for (int i = 0; i < notFilters.size(); i++) {
if (res == null) {
- res = new OpenBitSetDISI(getDISI(notFilters, i, reader), reader.maxDoc());
+ res = new OpenBitSetDISI(getDISI(notFilters, i, context), reader.maxDoc());
res.flip(0, reader.maxDoc()); // NOTE: may set bits on deleted docs
} else {
- DocIdSet dis = notFilters.get(i).getDocIdSet(reader);
+ DocIdSet dis = notFilters.get(i).getDocIdSet(context);
if(dis instanceof OpenBitSet) {
// optimized case for OpenBitSets
res.andNot((OpenBitSet) dis);
} else {
- res.inPlaceNot(getDISI(notFilters, i, reader));
+ res.inPlaceNot(getDISI(notFilters, i, context));
}
}
}
@@ -92,14 +93,14 @@ public class BooleanFilter extends Filte
if (mustFilters!=null) {
for (int i = 0; i < mustFilters.size(); i++) {
if (res == null) {
- res = new OpenBitSetDISI(getDISI(mustFilters, i, reader), reader.maxDoc());
+ res = new OpenBitSetDISI(getDISI(mustFilters, i, context), reader.maxDoc());
} else {
- DocIdSet dis = mustFilters.get(i).getDocIdSet(reader);
+ DocIdSet dis = mustFilters.get(i).getDocIdSet(context);
if(dis instanceof OpenBitSet) {
// optimized case for OpenBitSets
res.and((OpenBitSet) dis);
} else {
- res.inPlaceAnd(getDISI(mustFilters, i, reader));
+ res.inPlaceAnd(getDISI(mustFilters, i, context));
}
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java Thu Jan 13 02:09:33 2011
@@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Similarity;
/**
* The BoostingQuery class can be used to effectively demote results that match a given query.
@@ -58,7 +58,7 @@ public class BoostingQuery extends Query
BooleanQuery result = new BooleanQuery() {
@Override
- public Similarity getSimilarity(Searcher searcher) {
+ public Similarity getSimilarity(IndexSearcher searcher) {
return new DefaultSimilarity() {
@Override
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/ChainedFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/ChainedFilter.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/ChainedFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/ChainedFilter.java Thu Jan 13 02:09:33 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
@@ -96,21 +97,21 @@ public class ChainedFilter extends Filte
* {@link Filter#getDocIdSet}.
*/
@Override
- public DocIdSet getDocIdSet(IndexReader reader) throws IOException
+ public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException
{
int[] index = new int[1]; // use array as reference to modifiable int;
index[0] = 0; // an object attribute would not be thread safe.
if (logic != -1)
- return getDocIdSet(reader, logic, index);
+ return getDocIdSet(context, logic, index);
else if (logicArray != null)
- return getDocIdSet(reader, logicArray, index);
+ return getDocIdSet(context, logicArray, index);
else
- return getDocIdSet(reader, DEFAULT, index);
+ return getDocIdSet(context, DEFAULT, index);
}
- private DocIdSetIterator getDISI(Filter filter, IndexReader reader)
+ private DocIdSetIterator getDISI(Filter filter, AtomicReaderContext context)
throws IOException {
- DocIdSet docIdSet = filter.getDocIdSet(reader);
+ DocIdSet docIdSet = filter.getDocIdSet(context);
if (docIdSet == null) {
return DocIdSet.EMPTY_DOCIDSET.iterator();
} else {
@@ -123,9 +124,10 @@ public class ChainedFilter extends Filte
}
}
- private OpenBitSetDISI initialResult(IndexReader reader, int logic, int[] index)
+ private OpenBitSetDISI initialResult(AtomicReaderContext context, int logic, int[] index)
throws IOException
{
+ IndexReader reader = context.reader;
OpenBitSetDISI result;
/**
* First AND operation takes place against a completely false
@@ -133,12 +135,12 @@ public class ChainedFilter extends Filte
*/
if (logic == AND)
{
- result = new OpenBitSetDISI(getDISI(chain[index[0]], reader), reader.maxDoc());
+ result = new OpenBitSetDISI(getDISI(chain[index[0]], context), reader.maxDoc());
++index[0];
}
else if (logic == ANDNOT)
{
- result = new OpenBitSetDISI(getDISI(chain[index[0]], reader), reader.maxDoc());
+ result = new OpenBitSetDISI(getDISI(chain[index[0]], context), reader.maxDoc());
result.flip(0,reader.maxDoc()); // NOTE: may set bits for deleted docs.
++index[0];
}
@@ -155,13 +157,13 @@ public class ChainedFilter extends Filte
* @param logic Logical operation
* @return DocIdSet
*/
- private DocIdSet getDocIdSet(IndexReader reader, int logic, int[] index)
+ private DocIdSet getDocIdSet(AtomicReaderContext context, int logic, int[] index)
throws IOException
{
- OpenBitSetDISI result = initialResult(reader, logic, index);
+ OpenBitSetDISI result = initialResult(context, logic, index);
for (; index[0] < chain.length; index[0]++)
{
- doChain(result, logic, chain[index[0]].getDocIdSet(reader));
+ doChain(result, logic, chain[index[0]].getDocIdSet(context));
}
return result;
}
@@ -172,16 +174,16 @@ public class ChainedFilter extends Filte
* @param logic Logical operation
* @return DocIdSet
*/
- private DocIdSet getDocIdSet(IndexReader reader, int[] logic, int[] index)
+ private DocIdSet getDocIdSet(AtomicReaderContext info, int[] logic, int[] index)
throws IOException
{
if (logic.length != chain.length)
throw new IllegalArgumentException("Invalid number of elements in logic array");
- OpenBitSetDISI result = initialResult(reader, logic[0], index);
+ OpenBitSetDISI result = initialResult(info, logic[0], index);
for (; index[0] < chain.length; index[0]++)
{
- doChain(result, logic[index[0]], chain[index[0]].getDocIdSet(reader));
+ doChain(result, logic[index[0]], chain[index[0]].getDocIdSet(info));
}
return result;
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java Thu Jan 13 02:09:33 2011
@@ -19,6 +19,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.TermsEnum;
@@ -27,7 +28,8 @@ import org.apache.lucene.util.OpenBitSet
import org.apache.lucene.util.Bits;
public class DuplicateFilter extends Filter
-{
+{ // TODO: make duplicate filter aware of ReaderContext such that we can
+ // filter duplicates across segments
String fieldName;
@@ -70,15 +72,15 @@ public class DuplicateFilter extends Fil
}
@Override
- public DocIdSet getDocIdSet(IndexReader reader) throws IOException
+ public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException
{
if(processingMode==PM_FAST_INVALIDATION)
{
- return fastBits(reader);
+ return fastBits(context.reader);
}
else
{
- return correctBits(reader);
+ return correctBits(context.reader);
}
}
@@ -96,7 +98,7 @@ public class DuplicateFilter extends Fil
} else {
docs = termsEnum.docs(delDocs, docs);
int doc = docs.nextDoc();
- if (doc != docs.NO_MORE_DOCS) {
+ if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KM_USE_FIRST_OCCURRENCE) {
bits.set(doc);
} else {
@@ -104,7 +106,7 @@ public class DuplicateFilter extends Fil
while (true) {
lastDoc = doc;
doc = docs.nextDoc();
- if (doc == docs.NO_MORE_DOCS) {
+ if (doc == DocsEnum.NO_MORE_DOCS) {
break;
}
}
@@ -136,7 +138,7 @@ public class DuplicateFilter extends Fil
// unset potential duplicates
docs = termsEnum.docs(delDocs, docs);
int doc = docs.nextDoc();
- if (doc != docs.NO_MORE_DOCS) {
+ if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KM_USE_FIRST_OCCURRENCE) {
doc = docs.nextDoc();
}
@@ -147,7 +149,7 @@ public class DuplicateFilter extends Fil
lastDoc = doc;
bits.clear(lastDoc);
doc = docs.nextDoc();
- if (doc == docs.NO_MORE_DOCS) {
+ if (doc == DocsEnum.NO_MORE_DOCS) {
break;
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Thu Jan 13 02:09:33 2011
@@ -292,7 +292,7 @@ public class FuzzyLikeThisQuery extends
{
//optimize where only one selected variant
ScoreTerm st= variants.get(0);
- TermQuery tq = new FuzzyTermQuery(st.term,ignoreTF);
+ Query tq = ignoreTF ? new ConstantScoreQuery(new TermQuery(st.term)) : new TermQuery(st.term, 1);
tq.setBoost(st.score); // set the boost to a mix of IDF and score
bq.add(tq, BooleanClause.Occur.SHOULD);
}
@@ -303,7 +303,8 @@ public class FuzzyLikeThisQuery extends
.hasNext();)
{
ScoreTerm st = iterator2.next();
- TermQuery tq = new FuzzyTermQuery(st.term,ignoreTF); // found a match
+ // found a match
+ Query tq = ignoreTF ? new ConstantScoreQuery(new TermQuery(st.term)) : new TermQuery(st.term, 1);
tq.setBoost(st.score); // set the boost using the ScoreTerm's score
termVariants.add(tq, BooleanClause.Occur.SHOULD); // add to query
}
@@ -348,45 +349,8 @@ public class FuzzyLikeThisQuery extends
return termA.score < termB.score;
}
- }
-
- //overrides basic TermQuery to negate effects of IDF (idf is factored into boost of containing BooleanQuery)
- private static class FuzzyTermQuery extends TermQuery
- {
- boolean ignoreTF;
- public FuzzyTermQuery(Term t, boolean ignoreTF)
- {
- super(t);
- this.ignoreTF=ignoreTF;
- }
- @Override
- public Similarity getSimilarity(Searcher searcher)
- {
- Similarity result = super.getSimilarity(searcher);
- result = new SimilarityDelegator(result) {
-
- @Override
- public float tf(float freq)
- {
- if(ignoreTF)
- {
- return 1; //ignore tf
- }
- return super.tf(freq);
- }
- @Override
- public float idf(int docFreq, int numDocs)
- {
- //IDF is already factored into individual term boosts
- return 1;
- }
- };
- return result;
- }
- }
+ }
-
-
/* (non-Javadoc)
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java Thu Jan 13 02:09:33 2011
@@ -23,6 +23,7 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Terms;
@@ -57,7 +58,8 @@ public class TermsFilter extends Filter
* @see org.apache.lucene.search.Filter#getDocIdSet(org.apache.lucene.index.IndexReader)
*/
@Override
- public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+ public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ IndexReader reader = context.reader;
OpenBitSet result=new OpenBitSet(reader.maxDoc());
Fields fields = reader.fields();
BytesRef br = new BytesRef();
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java Thu Jan 13 02:09:33 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.Term;
@@ -83,7 +84,7 @@ public class BooleanFilterTest extends L
private void tstFilterCard(String mes, int expected, Filter filt)
throws Throwable
{
- DocIdSetIterator disi = filt.getDocIdSet(reader).iterator();
+ DocIdSetIterator disi = filt.getDocIdSet(new AtomicReaderContext(reader)).iterator();
int actual = 0;
while (disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
actual++;
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/ChainedFilterTest.java Thu Jan 13 02:09:33 2011
@@ -29,11 +29,10 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryWrapperFilter;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.search.TopDocs;
@@ -195,7 +194,7 @@ public class ChainedFilterTest extends L
IndexReader reader = writer.getReader();
writer.close();
- Searcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = new IndexSearcher(reader);
Query query = new TermQuery(new Term("none", "none"));
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Thu Jan 13 02:09:33 2011
@@ -21,6 +21,7 @@ import java.util.HashSet;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
@@ -59,23 +60,26 @@ public class TermsFilterTest extends Luc
w.addDocument(doc);
}
IndexReader reader = new SlowMultiReaderWrapper(w.getReader());
+ assertTrue(reader.getTopReaderContext().isAtomic);
+ AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext();
+ assertTrue(context.isAtomic);
w.close();
TermsFilter tf=new TermsFilter();
tf.addTerm(new Term(fieldName,"19"));
- OpenBitSet bits = (OpenBitSet)tf.getDocIdSet(reader);
+ OpenBitSet bits = (OpenBitSet)tf.getDocIdSet(context);
assertEquals("Must match nothing", 0, bits.cardinality());
tf.addTerm(new Term(fieldName,"20"));
- bits = (OpenBitSet)tf.getDocIdSet(reader);
+ bits = (OpenBitSet)tf.getDocIdSet(context);
assertEquals("Must match 1", 1, bits.cardinality());
tf.addTerm(new Term(fieldName,"10"));
- bits = (OpenBitSet)tf.getDocIdSet(reader);
+ bits = (OpenBitSet)tf.getDocIdSet(context);
assertEquals("Must match 2", 2, bits.cardinality());
tf.addTerm(new Term(fieldName,"00"));
- bits = (OpenBitSet)tf.getDocIdSet(reader);
+ bits = (OpenBitSet)tf.getDocIdSet(context);
assertEquals("Must match 2", 2, bits.cardinality());
reader.close();
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java Thu Jan 13 02:09:33 2011
@@ -27,10 +27,8 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.spans.SpanFirstQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@@ -85,33 +83,6 @@ public class TestSpanRegexQuery extends
directory.close();
}
- public void testSpanRegexBug() throws CorruptIndexException, IOException {
- createRAMDirectories();
-
- SpanQuery srq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "a.*")));
- SpanQuery stq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "b.*")));
- SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { srq, stq }, 6,
- true);
-
- // 1. Search the same store which works
- IndexSearcher[] arrSearcher = new IndexSearcher[2];
- arrSearcher[0] = new IndexSearcher(indexStoreA, true);
- arrSearcher[1] = new IndexSearcher(indexStoreB, true);
- MultiSearcher searcher = new MultiSearcher(arrSearcher);
- int numHits = searcher.search(query, null, 1000).totalHits;
- arrSearcher[0].close();
- arrSearcher[1].close();
-
- // Will fail here
- // We expect 2 but only one matched
- // The rewriter function only write it once on the first IndexSearcher
- // So it's using term: a1 b1 to search on the second IndexSearcher
- // As a result, it won't match the document in the second IndexSearcher
- assertEquals(2, numHits);
- indexStoreA.close();
- indexStoreB.close();
- }
-
private void createRAMDirectories() throws CorruptIndexException,
LockObtainFailedException, IOException {
// creating a document to store
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java?rev=1058390&r1=1058389&r2=1058390&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/BooleanQueryTst.java Thu Jan 13 02:09:33 2011
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Query;
@@ -122,7 +121,7 @@ public class BooleanQueryTst {
/* if (verbose) System.out.println("Lucene: " + query.toString()); */
TestCollector tc = new TestCollector();
- Searcher searcher = new IndexSearcher(dBase.getDb(), true);
+ IndexSearcher searcher = new IndexSearcher(dBase.getDb(), true);
try {
searcher.search(query, tc);
} finally {