You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/18 23:28:20 UTC
svn commit: r1233096 [11/13] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/analysis/kuromoji/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/maven/modules/...
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Wed Jan 18 22:28:07 2012
@@ -23,6 +23,7 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
@@ -87,12 +88,12 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getDoubles(readerA, "theDouble", false);
- cache.getDoubles(readerA, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
- cache.getDoubles(readerB, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
+ cache.getDoubles(new SlowMultiReaderWrapper(readerA), "theDouble", false);
+ cache.getDoubles(new SlowMultiReaderWrapper(readerA), "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
+ cache.getDoubles(new SlowMultiReaderWrapper(readerB), "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
- cache.getInts(readerX, "theInt", false);
- cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
+ cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", false);
+ cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", FieldCache.DEFAULT_INT_PARSER, false);
// // //
@@ -110,9 +111,9 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
- cache.getTerms(readerX, "theInt");
- cache.getBytes(readerX, "theByte", false);
+ cache.getInts(new SlowMultiReaderWrapper(readerX), "theInt", FieldCache.DEFAULT_INT_PARSER, false);
+ cache.getTerms(new SlowMultiReaderWrapper(readerX), "theInt");
+ cache.getBytes(new SlowMultiReaderWrapper(readerX), "theByte", false);
// // //
@@ -134,11 +135,11 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getTerms(readerA, "theString");
- cache.getTerms(readerB, "theString");
- cache.getTerms(readerX, "theString");
+ cache.getTerms(new SlowMultiReaderWrapper(readerA), "theString");
+ cache.getTerms(new SlowMultiReaderWrapper(readerB), "theString");
+ cache.getTerms(new SlowMultiReaderWrapper(readerX), "theString");
- cache.getBytes(readerX, "theByte", false);
+ cache.getBytes(new SlowMultiReaderWrapper(readerX), "theByte", false);
// // //
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Wed Jan 18 22:28:07 2012
@@ -1050,6 +1050,7 @@ public class TestFSTs extends LuceneTest
}
Terms terms = MultiFields.getTerms(r, "body");
if (terms != null) {
+ final IntsRef scratchIntsRef = new IntsRef();
final TermsEnum termsEnum = terms.iterator(null);
if (VERBOSE) {
System.out.println("TEST: got termsEnum=" + termsEnum);
@@ -1073,7 +1074,7 @@ public class TestFSTs extends LuceneTest
} else {
output = termsEnum.docFreq();
}
- builder.add(term, outputs.get(output));
+ builder.add(Util.toIntsRef(term, scratchIntsRef), outputs.get(output));
ord++;
if (VERBOSE && ord % 100000 == 0 && LuceneTestCase.TEST_NIGHTLY) {
System.out.println(ord + " terms...");
@@ -1373,7 +1374,7 @@ public class TestFSTs extends LuceneTest
public void testSingleString() throws Exception {
final Outputs<Object> outputs = NoOutputs.getSingleton();
final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, outputs);
- b.add(new BytesRef("foobar"), outputs.getNoOutput());
+ b.add(Util.toIntsRef(new BytesRef("foobar"), new IntsRef()), outputs.getNoOutput());
final BytesRefFSTEnum<Object> fstEnum = new BytesRefFSTEnum<Object>(b.finish());
assertNull(fstEnum.seekFloor(new BytesRef("foo")));
assertNull(fstEnum.seekCeil(new BytesRef("foobaz")));
@@ -1395,9 +1396,9 @@ public class TestFSTs extends LuceneTest
final BytesRef b = new BytesRef("b");
final BytesRef c = new BytesRef("c");
- builder.add(a, outputs.get(17));
- builder.add(b, outputs.get(42));
- builder.add(c, outputs.get(13824324872317238L));
+ builder.add(Util.toIntsRef(a, new IntsRef()), outputs.get(17));
+ builder.add(Util.toIntsRef(b, new IntsRef()), outputs.get(42));
+ builder.add(Util.toIntsRef(c, new IntsRef()), outputs.get(13824324872317238L));
final FST<Long> fst = builder.finish();
@@ -1628,13 +1629,14 @@ public class TestFSTs extends LuceneTest
int line = 0;
final BytesRef term = new BytesRef();
+ final IntsRef scratchIntsRef = new IntsRef();
while (line < lines.length) {
String w = lines[line++];
if (w == null) {
break;
}
term.copyChars(w);
- b.add(term, nothing);
+ b.add(Util.toIntsRef(term, scratchIntsRef), nothing);
}
return b.finish();
@@ -1698,8 +1700,8 @@ public class TestFSTs extends LuceneTest
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null);
- builder.add("stat", outputs.get(17));
- builder.add("station", outputs.get(10));
+ builder.add(Util.toUTF32("stat", new IntsRef()), outputs.get(17));
+ builder.add(Util.toUTF32("station", new IntsRef()), outputs.get(10));
final FST<Long> fst = builder.finish();
//Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
StringWriter w = new StringWriter();
@@ -1713,8 +1715,8 @@ public class TestFSTs extends LuceneTest
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null);
- builder.add(new BytesRef("stat"), outputs.getNoOutput());
- builder.add(new BytesRef("station"), outputs.getNoOutput());
+ builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRef()), outputs.getNoOutput());
+ builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRef()), outputs.getNoOutput());
final FST<Long> fst = builder.finish();
StringWriter w = new StringWriter();
//Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
Modified: lucene/dev/branches/solrcloud/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/CHANGES.txt?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/CHANGES.txt Wed Jan 18 22:28:07 2012
@@ -12,19 +12,6 @@ API Changes
* LUCENE-2413: Removed the AnalyzerUtil in common/miscellaneous. (Robert Muir)
- * LUCENE-2167,LUCENE-2699,LUCENE-2763,LUCENE-2847: StandardTokenizer/Analyzer
- in common/standard/ now implement the Word Break rules from the Unicode 6.0.0
- Text Segmentation algorithm (UAX#29), covering the full range of Unicode code
- points, including values from U+FFFF to U+10FFFF
-
- ClassicTokenizer/Analyzer retains the old (pre-Lucene 3.1) StandardTokenizer/
- Analyzer implementation and behavior. Only the Unicode Basic Multilingual
- Plane (code points from U+0000 to U+FFFF) is covered.
-
- UAX29URLEmailTokenizer tokenizes URLs and E-mail addresses according to the
- relevant RFCs, in addition to implementing the UAX#29 Word Break rules.
- (Steven Rowe, Robert Muir, Uwe Schindler)
-
* LUCENE-1370: Added ShingleFilter option to output unigrams if no shingles
can be generated. (Chris Harris via Steven Rowe)
@@ -48,7 +35,7 @@ New Features
(accurate stemmer) for Polish (includes morphosyntactic annotations).
(MichaÅ DybizbaÅski, Dawid Weiss)
- * LUCENE-2413: Consolidated Solr analysis components into common.
+ * LUCENE-2413: Consolidated Lucene/Solr analysis components into common.
New features from Solr now available to Lucene users include:
- o.a.l.analysis.commongrams: Constructs n-grams for frequently occurring terms
and phrases.
@@ -73,7 +60,7 @@ New Features
- o.a.l.analysis.phonetic: Package for phonetic search, containing various
phonetic encoders such as Double Metaphone.
- * LUCENE-2413: Consolidated all Lucene analyzers into common.
+ Some existing analysis components changed packages:
- o.a.l.analysis.KeywordAnalyzer -> o.a.l.analysis.core.KeywordAnalyzer
- o.a.l.analysis.KeywordTokenizer -> o.a.l.analysis.core.KeywordTokenizer
- o.a.l.analysis.LetterTokenizer -> o.a.l.analysis.core.LetterTokenizer
@@ -103,16 +90,6 @@ New Features
- o.a.l.analysis.CharTokenizer -> o.a.l.analysis.util.CharTokenizer
- o.a.l.util.CharacterUtils -> o.a.l.analysis.util.CharacterUtils
- * SOLR-1057: Add PathHierarchyTokenizer that represents file path hierarchies as synonyms of
- /something, /something/something, /something/something/else. (Ryan McKinley, Koji Sekiguchi)
-
- * LUCENE-3414: Added HunspellStemFilter which uses a provided pure Java implementation of the
- Hunspell algorithm. (Chris Male)
-
-Build
-
- * LUCENE-2413: All analyzers in contrib/analyzers and contrib/icu were moved to the
- analysis module. The 'smartcn' and 'stempel' components now depend on 'common'.
- (Robert Muir)
-
- * LUCENE-3376: Moved ReusableAnalyzerBase into lucene core. (Chris Male)
+ All analyzers in contrib/analyzers and contrib/icu were moved to the
+ analysis module. The 'smartcn' and 'stempel' components now depend on 'common'.
+ (Chris Male, Robert Muir)
Modified: lucene/dev/branches/solrcloud/modules/analysis/NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/NOTICE.txt?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/NOTICE.txt (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/NOTICE.txt Wed Jan 18 22:28:07 2012
@@ -71,3 +71,86 @@ LGPL and Creative Commons ShareAlike.
Morfologic includes data from BSD-licensed dictionary of Polish (SGJP)
(http://sgjp.pl/morfeusz/)
+
+===========================================================================
+Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration
+===========================================================================
+
+This software includes a binary and/or source version of data from
+
+ mecab-ipadic-2.7.0-20070801
+
+which can be obtained from
+
+ http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz
+
+or
+
+ http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz
+
+===========================================================================
+mecab-ipadic-2.7.0-20070801 Notice
+===========================================================================
+
+Nara Institute of Science and Technology (NAIST),
+the copyright holders, disclaims all warranties with regard to this
+software, including all implied warranties of merchantability and
+fitness, in no event shall NAIST be liable for
+any special, indirect or consequential damages or any damages
+whatsoever resulting from loss of use, data or profits, whether in an
+action of contract, negligence or other tortuous action, arising out
+of or in connection with the use or performance of this software.
+
+A large portion of the dictionary entries
+originate from ICOT Free Software. The following conditions for ICOT
+Free Software applies to the current dictionary as well.
+
+Each User may also freely distribute the Program, whether in its
+original form or modified, to any third party or parties, PROVIDED
+that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+on, or be attached to, the Program, which is distributed substantially
+in the same form as set out herein and that such intended
+distribution, if actually made, will neither violate or otherwise
+contravene any of the laws and regulations of the countries having
+jurisdiction over the User or the intended distribution itself.
+
+NO WARRANTY
+
+The program was produced on an experimental basis in the course of the
+research and development conducted during the project and is provided
+to users as so produced on an experimental basis. Accordingly, the
+program is provided without any warranty whatsoever, whether express,
+implied, statutory or otherwise. The term "warranty" used herein
+includes, but is not limited to, any warranty of the quality,
+performance, merchantability and fitness for a particular purpose of
+the program and the nonexistence of any infringement or violation of
+any right of any third party.
+
+Each user of the program will agree and understand, and be deemed to
+have agreed and understood, that there is no warranty whatsoever for
+the program and, accordingly, the entire risk arising from or
+otherwise connected with the program is assumed by the user.
+
+Therefore, neither ICOT, the copyright holder, or any other
+organization that participated in or was otherwise related to the
+development of the program and their respective officials, directors,
+officers and other employees shall be held liable for any and all
+damages, including, without limitation, general, special, incidental
+and consequential damages, arising out of or otherwise in connection
+with the use or inability to use the program or any product, material
+or result produced or otherwise obtained by using the program,
+regardless of whether they have been advised of, or otherwise had
+knowledge of, the possibility of such damages at any time during the
+project or thereafter. Each user will be deemed to have agreed to the
+foregoing by his or her commencement of use of the program. The term
+"use" as used herein includes, but is not limited to, the use,
+modification, copying and distribution of the program and the
+production of secondary products from the program.
+
+In the case where the program, whether in its original form or
+modified, was distributed or delivered to or received by a user from
+any person, organization or entity other than ICOT, unless it makes or
+grants independently of ICOT any specific warranty to the user in
+writing, such person, organization or entity, will also be exempted
+from and not be held liable to the user for any such damages as noted
+above as far as the program is concerned.
Modified: lucene/dev/branches/solrcloud/modules/analysis/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/README.txt?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/README.txt (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/README.txt Wed Jan 18 22:28:07 2012
@@ -22,6 +22,12 @@ lucene-analyzers-icu-XX.jar
International Components for Unicode (ICU). Note: this module depends on
the ICU4j jar file (version >= 4.6.0)
+lucene-analyzers-kuromoji-XX.jar
+ An analyzer with morphological analysis for Japanese.
+
+lucene-analyzers-morfologik-XX.jar
+ An analyzer using the Morfologik stemming library.
+
lucene-analyzers-phonetic-XX.jar
An add-on analysis library that provides phonetic encoders via Apache
Commons-Codec. Note: this module depends on the commons-codec jar
@@ -35,21 +41,20 @@ lucene-analyzers-stempel-XX.jar
An add-on analysis library that contains a universal algorithmic stemmer,
including tables for the Polish language.
-lucene-analyzers-morfologik-XX.jar
- An analyzer using the Morfologik stemming library.
-
common/src/java
icu/src/java
+kuromoji/src/java
+morfologik/src/java
phonetic/src/java
smartcn/src/java
stempel/src/java
-morfologik/src/java
- The source code for the ffve libraries.
+ The source code for the libraries.
common/src/test
icu/src/test
+kuromoji/src/test
+morfologik/src/test
phonetic/src/test
smartcn/src/test
stempel/src/test
-morfologik/src/test
- Unit tests for the five libraries.
+ Unit tests for the libraries.
Modified: lucene/dev/branches/solrcloud/modules/analysis/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/build.xml?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/build.xml (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/build.xml Wed Jan 18 22:28:07 2012
@@ -23,9 +23,10 @@
Additional Analyzers
- common: Additional Analyzers
- icu: Analyzers that use functionality from ICU
+ - kuromoji: Japanese Morphological Analyzer
+ - morfologik: Morfologik Stemmer
- smartcn: Smart Analyzer for Simplified Chinese Text
- stempel: Algorithmic Stemmer for Polish
- - morfologik: Morfologik Stemmer
</description>
<target name="common">
@@ -36,6 +37,14 @@
<ant dir="icu" />
</target>
+ <target name="kuromoji">
+ <ant dir="kuromoji" />
+ </target>
+
+ <target name="morfologik">
+ <ant dir="morfologik" />
+ </target>
+
<target name="phonetic">
<ant dir="phonetic" />
</target>
@@ -48,52 +57,53 @@
<ant dir="stempel" />
</target>
- <target name="morfologik">
- <ant dir="morfologik" />
- </target>
-
<target name="default" depends="compile"/>
- <target name="compile" depends="common,icu,phonetic,smartcn,stempel,morfologik" />
+ <target name="compile" depends="common,icu,kuromoji,morfologik,phonetic,smartcn,stempel" />
<target name="clean">
<ant dir="common" target="clean" />
<ant dir="icu" target="clean" />
+ <ant dir="kuromoji" target="clean"/>
+ <ant dir="morfologik" target="clean" />
<ant dir="phonetic" target="clean" />
<ant dir="smartcn" target="clean" />
<ant dir="stempel" target="clean" />
- <ant dir="morfologik" target="clean" />
</target>
<target name="validate">
<ant dir="common" target="validate" />
<ant dir="icu" target="validate" />
+ <ant dir="kuromoji" target="validate" />
+ <ant dir="morfologik" target="validate" />
<ant dir="phonetic" target="validate" />
<ant dir="smartcn" target="validate" />
<ant dir="stempel" target="validate" />
- <ant dir="morfologik" target="validate" />
</target>
<target name="compile-core">
<ant dir="common" target="compile-core" />
<ant dir="icu" target="compile-core" />
+ <ant dir="kuromoji" target="compile-core" />
+ <ant dir="morfologik" target="compile-core" />
<ant dir="phonetic" target="compile-core" />
<ant dir="smartcn" target="compile-core" />
<ant dir="stempel" target="compile-core" />
- <ant dir="morfologik" target="compile-core" />
</target>
<target name="compile-test">
<ant dir="common" target="compile-test" />
<ant dir="icu" target="compile-test" />
+ <ant dir="kuromoji" target="compile-test" />
+ <ant dir="morfologik" target="compile-test" />
<ant dir="phonetic" target="compile-test" />
<ant dir="smartcn" target="compile-test" />
<ant dir="stempel" target="compile-test" />
- <ant dir="morfologik" target="compile-test" />
</target>
<target name="test">
<ant dir="common" target="test" />
<ant dir="icu" target="test" />
+ <ant dir="kuromoji" target="test" />
+ <ant dir="morfologik" target="test" />
<ant dir="phonetic" target="test" />
<ant dir="smartcn" target="test" />
<ant dir="stempel" target="test" />
- <ant dir="morfologik" target="test" />
</target>
<target name="build-artifacts-and-tests" depends="default,compile-test" />
@@ -101,28 +111,31 @@
<target name="dist-maven" depends="default,javadocs">
<ant dir="common" target="dist-maven" />
<ant dir="icu" target="dist-maven" />
+ <ant dir="kuromoji" target="dist-maven" />
+ <ant dir="morfologik" target="dist-maven" />
<ant dir="phonetic" target="dist-maven" />
<ant dir="smartcn" target="dist-maven" />
<ant dir="stempel" target="dist-maven" />
- <ant dir="morfologik" target="dist-maven" />
</target>
<target name="javadocs">
<ant dir="common" target="javadocs" />
<ant dir="icu" target="javadocs" />
+ <ant dir="kuromoji" target="javadocs" />
+ <ant dir="morfologik" target="javadocs" />
<ant dir="phonetic" target="javadocs" />
<ant dir="smartcn" target="javadocs" />
<ant dir="stempel" target="javadocs" />
- <ant dir="morfologik" target="javadocs" />
</target>
<target name="javadocs-index.html">
<ant dir="common" target="javadocs-index.html" />
<ant dir="icu" target="javadocs-index.html" />
+ <ant dir="kuromoji" target="javadocs-index.html" />
+ <ant dir="morfologik" target="javadocs-index.html" />
<ant dir="phonetic" target="javadocs-index.html" />
<ant dir="smartcn" target="javadocs-index.html" />
<ant dir="stempel" target="javadocs-index.html" />
- <ant dir="morfologik" target="javadocs-index.html" />
</target>
</project>
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java Wed Jan 18 22:28:07 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.hunspell.HunspellStemmer.Stem;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
/**
@@ -34,6 +35,7 @@ public final class HunspellStemFilter ex
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final HunspellStemmer stemmer;
private List<Stem> buffer;
@@ -84,6 +86,10 @@ public final class HunspellStemFilter ex
return false;
}
+ if (keywordAtt.isKeyword()) {
+ return true;
+ }
+
buffer = dedup ? stemmer.uniqueStems(termAtt.buffer(), termAtt.length()) : stemmer.stem(termAtt.buffer(), termAtt.length());
if (buffer.isEmpty()) { // we do not know this word, return it unchanged
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Wed Jan 18 22:28:07 2012
@@ -16,21 +16,22 @@ package org.apache.lucene.analysis.query
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.AnalyzerWrapper;
+import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.BytesRef;
-
-import java.io.IOException;
-import java.util.*;
/**
* An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection
@@ -84,7 +85,7 @@ public final class QueryAutoStopWordAnal
Analyzer delegate,
IndexReader indexReader,
int maxDocFreq) throws IOException {
- this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxDocFreq);
+ this(matchVersion, delegate, indexReader, ReaderUtil.getIndexedFields(indexReader), maxDocFreq);
}
/**
@@ -104,7 +105,7 @@ public final class QueryAutoStopWordAnal
Analyzer delegate,
IndexReader indexReader,
float maxPercentDocs) throws IOException {
- this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxPercentDocs);
+ this(matchVersion, delegate, indexReader, ReaderUtil.getIndexedFields(indexReader), maxPercentDocs);
}
/**
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Wed Jan 18 22:28:07 2012
@@ -33,9 +33,11 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.Util;
/**
* A map of synonyms, keys and values are phrases.
@@ -262,6 +264,8 @@ public class SynonymMap {
Set<CharsRef> keys = workingSet.keySet();
CharsRef sortedKeys[] = keys.toArray(new CharsRef[keys.size()]);
Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());
+
+ final IntsRef scratchIntsRef = new IntsRef();
//System.out.println("fmap.build");
for (int keyIdx = 0; keyIdx < sortedKeys.length; keyIdx++) {
@@ -307,7 +311,7 @@ public class SynonymMap {
scratch.length = scratchOutput.getPosition() - scratch.offset;
//System.out.println(" add input=" + input + " output=" + scratch + " offset=" + scratch.offset + " length=" + scratch.length + " count=" + count);
- builder.add(input, BytesRef.deepCopyOf(scratch));
+ builder.add(Util.toUTF32(input, scratchIntsRef), BytesRef.deepCopyOf(scratch));
}
FST<BytesRef> fst = builder.finish();
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java Wed Jan 18 22:28:07 2012
@@ -283,7 +283,8 @@ public class TestClassicAnalyzer extends
DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
"content",
- new BytesRef("another"));
+ new BytesRef("another"),
+ false);
assertTrue(tps.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, tps.freq());
assertEquals(3, tps.nextPosition());
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java Wed Jan 18 22:28:07 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.analysis.hunspell;
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
import org.junit.Test;
@@ -26,14 +27,14 @@ import java.text.ParseException;
import static junit.framework.Assert.assertEquals;
-public class HunspellDictionaryTest {
+public class HunspellDictionaryTest extends LuceneTestCase {
@Test
public void testHunspellDictionary_loadDicAff() throws IOException, ParseException {
InputStream affixStream = getClass().getResourceAsStream("test.aff");
InputStream dictStream = getClass().getResourceAsStream("test.dic");
- HunspellDictionary dictionary = new HunspellDictionary(affixStream, dictStream, Version.LUCENE_40);
+ HunspellDictionary dictionary = new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java Wed Jan 18 22:28:07 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.analysis.hunspell;
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +29,7 @@ import java.util.List;
import static junit.framework.Assert.assertEquals;
-public class HunspellStemmerTest {
+public class HunspellStemmerTest extends LuceneTestCase {
private static HunspellStemmer stemmer;
@@ -116,7 +117,7 @@ public class HunspellStemmerTest {
InputStream affixStream = HunspellStemmerTest.class.getResourceAsStream("test.aff");
InputStream dictStream = HunspellStemmerTest.class.getResourceAsStream("test.dic");
- HunspellDictionary dictionary = new HunspellDictionary(affixStream, dictStream, Version.LUCENE_40, ignoreCase);
+ HunspellDictionary dictionary = new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT, ignoreCase);
stemmer = new HunspellStemmer(dictionary);
affixStream.close();
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Wed Jan 18 22:28:07 2012
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.core.L
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -110,16 +109,15 @@ public class TestTeeSinkTokenFilter exte
TermsEnum termsEnum = vector.iterator(null);
termsEnum.next();
assertEquals(2, termsEnum.totalTermFreq());
- DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null);
- OffsetAttribute offsetAtt = positions.attributes().getAttribute(OffsetAttribute.class);
+ DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null, true);
assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(2, positions.freq());
positions.nextPosition();
- assertEquals(0, offsetAtt.startOffset());
- assertEquals(4, offsetAtt.endOffset());
+ assertEquals(0, positions.startOffset());
+ assertEquals(4, positions.endOffset());
positions.nextPosition();
- assertEquals(8, offsetAtt.startOffset());
- assertEquals(12, offsetAtt.endOffset());
+ assertEquals(8, positions.startOffset());
+ assertEquals(12, positions.endOffset());
assertEquals(DocsEnum.NO_MORE_DOCS, positions.nextDoc());
r.close();
dir.close();
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Wed Jan 18 22:28:07 2012
@@ -260,9 +260,16 @@ public class TestSynonymMapFilter extend
} else {
outputs[matchIDX] = outputs[matchIDX] + "/" + synOutputs[synUpto++];
}
- if (synOutputs.length == 1) {
- // Add endOffset
- outputs[matchIDX] = outputs[matchIDX] + ":" + ((inputIDX*2) + syn.in.length());
+ final int endOffset;
+ if (matchIDX < numInputs) {
+ if (synOutputs.length == 1) {
+ // Add full endOffset
+ endOffset = (inputIDX*2) + syn.in.length();
+ } else {
+ // Add endOffset matching input token's
+ endOffset = (matchIDX*2) + 1;
+ }
+ outputs[matchIDX] = outputs[matchIDX] + ":" + endOffset;
}
}
}
Modified: lucene/dev/branches/solrcloud/modules/analysis/icu/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/icu/build.xml?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/icu/build.xml (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/icu/build.xml Wed Jan 18 22:28:07 2012
@@ -85,6 +85,9 @@ are part of the ICU4C package. See http:
<path refid="additional.dependencies"/>
<pathelement location="${build.dir}/classes/tools"/>
</classpath>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ </assertions>
<arg value="${rbbi.src.dir}"/>
<arg value="${rbbi.dst.dir}"/>
</java>
@@ -104,6 +107,9 @@ are part of the ICU4C package. See http:
<path refid="additional.dependencies"/>
<pathelement location="${build.dir}/classes/tools"/>
</classpath>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ </assertions>
</java>
</target>
Modified: lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java (original)
+++ lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java Wed Jan 18 22:28:07 2012
@@ -109,8 +109,8 @@ public class DocMaker implements Closeab
fields.put(ID_FIELD, new Field(ID_FIELD, "", StringField.TYPE_STORED));
fields.put(NAME_FIELD, new Field(NAME_FIELD, "", ft));
- numericFields.put(DATE_MSEC_FIELD, new NumericField(DATE_MSEC_FIELD));
- numericFields.put(TIME_SEC_FIELD, new NumericField(TIME_SEC_FIELD));
+ numericFields.put(DATE_MSEC_FIELD, new NumericField(DATE_MSEC_FIELD, 0L));
+ numericFields.put(TIME_SEC_FIELD, new NumericField(TIME_SEC_FIELD, 0));
doc = new Document();
} else {
@@ -138,15 +138,34 @@ public class DocMaker implements Closeab
return f;
}
- NumericField getNumericField(String name) {
- if (!reuseFields) {
- return new NumericField(name);
+ NumericField getNumericField(String name, NumericField.DataType type) {
+ NumericField f;
+ if (reuseFields) {
+ f = numericFields.get(name);
+ } else {
+ f = null;
}
-
- NumericField f = numericFields.get(name);
+
if (f == null) {
- f = new NumericField(name);
- numericFields.put(name, f);
+ switch(type) {
+ case INT:
+ f = new NumericField(name, 0);
+ break;
+ case LONG:
+ f = new NumericField(name, 0L);
+ break;
+ case FLOAT:
+ f = new NumericField(name, 0.0f);
+ break;
+ case DOUBLE:
+ f = new NumericField(name, 0.0);
+ break;
+ default:
+ assert false;
+ }
+ if (reuseFields) {
+ numericFields.put(name, f);
+ }
}
return f;
}
@@ -249,15 +268,15 @@ public class DocMaker implements Closeab
date = new Date();
}
- NumericField dateField = ds.getNumericField(DATE_MSEC_FIELD);
- dateField.setLongValue(date.getTime());
+ NumericField dateField = ds.getNumericField(DATE_MSEC_FIELD, NumericField.DataType.LONG);
+ dateField.setValue(date.getTime());
doc.add(dateField);
util.cal.setTime(date);
final int sec = util.cal.get(Calendar.HOUR_OF_DAY)*3600 + util.cal.get(Calendar.MINUTE)*60 + util.cal.get(Calendar.SECOND);
- NumericField timeSecField = ds.getNumericField(TIME_SEC_FIELD);
- timeSecField.setIntValue(sec);
+ NumericField timeSecField = ds.getNumericField(TIME_SEC_FIELD, NumericField.DataType.INT);
+ timeSecField.setValue(sec);
doc.add(timeSecField);
// Set TITLE_FIELD
Modified: lucene/dev/branches/solrcloud/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/branches/solrcloud/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Wed Jan 18 22:28:07 2012
@@ -51,6 +51,7 @@ import org.apache.lucene.index.LogMergeP
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache.DocTermsIndex;
@@ -332,7 +333,7 @@ public class TestPerfTasksLogic extends
Benchmark benchmark = execBenchmark(algLines);
IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory());
- DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(r, "country");
+ DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(r), "country");
final int maxDoc = r.maxDoc();
assertEquals(1000, maxDoc);
BytesRef br = new BytesRef();
Modified: lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java (original)
+++ lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java Wed Jan 18 22:28:07 2012
@@ -47,24 +47,24 @@ import org.apache.lucene.facet.taxonomy.
* @lucene.experimental
*/
public interface CategoryEnhancement {
-
+
/**
* Get the bytes to be added to the category token payload for this
* enhancement.
* <p>
- * <b>NOTE</b>: The returned array is copied, it is recommended to allocate
- * a new one each time.
+ * <b>NOTE</b>: The returned array is copied, it is recommended to allocate a
+ * new one each time.
* <p>
* The bytes generated by this method are the input of
* {@link #extractCategoryTokenData(byte[], int, int)}.
*
* @param categoryAttribute
- * The attribute of the category.
+ * The attribute of the category.
* @return The bytes to be added to the category token payload for this
* enhancement.
*/
byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute);
-
+
/**
* Get the data of this enhancement from a category token payload.
* <p>
@@ -72,56 +72,61 @@ public interface CategoryEnhancement {
* {@link #getCategoryTokenBytes(CategoryAttribute)}.
*
* @param buffer
- * The payload buffer.
+ * The payload buffer.
* @param offset
- * The offset of this enhancement's data in the buffer.
+ * The offset of this enhancement's data in the buffer.
* @param length
- * The length of this enhancement's data (bytes).
+ * The length of this enhancement's data (bytes).
* @return An Object containing the data.
*/
Object extractCategoryTokenData(byte[] buffer, int offset, int length);
-
+
/**
- * Declarative method to indicate whether this enhancement generates
- * separate category list.
+ * Declarative method to indicate whether this enhancement generates separate
+ * category list.
*
* @return {@code true} if generates category list, else {@code false}.
*/
boolean generatesCategoryList();
-
+
/**
* Returns the text of this enhancement's category list term.
*
* @return The text of this enhancement's category list term.
*/
String getCategoryListTermText();
-
+
/**
- * Get the {@link CategoryListTokenizer} which generates the category list
- * for this enhancement. If {@link #generatesCategoryList()} returns
- * {@code false} this method will not be called.
+ * Get the {@link CategoryListTokenizer} which generates the category list for
+ * this enhancement. If {@link #generatesCategoryList()} returns {@code false}
+ * this method will not be called.
*
* @param tokenizer
- * The input stream containing categories.
+ * The input stream containing categories.
* @param indexingParams
- * The indexing params to use.
+ * The indexing params to use.
* @param taxonomyWriter
- * The taxonomy to add categories and get their ordinals.
+ * The taxonomy to add categories and get their ordinals.
* @return A {@link CategoryListTokenizer} generating the category list for
* this enhancement, with {@code tokenizer} as it's input.
*/
CategoryListTokenizer getCategoryListTokenizer(TokenStream tokenizer,
- EnhancementsIndexingParams indexingParams,
- TaxonomyWriter taxonomyWriter);
-
+ EnhancementsIndexingParams indexingParams, TaxonomyWriter taxonomyWriter);
+
/**
* Get a {@link CategoryProperty} class to be retained when creating
* {@link CategoryParentsStream}.
*
* @return the {@link CategoryProperty} class to be retained when creating
- * {@link CategoryParentsStream}, or {@code null} if there is no
- * such property.
+ * {@link CategoryParentsStream}, or {@code null} if there is no such
+ * property.
*/
Class<? extends CategoryProperty> getRetainableProperty();
-
+
+ /**
+ * Category enhancements must override {@link Object#equals(Object)}, as it is
+ * used in
+ * {@link EnhancementsPayloadIterator#getCategoryData(CategoryEnhancement)}.
+ */
+ public boolean equals(Object o);
}
Modified: lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java (original)
+++ lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java Wed Jan 18 22:28:07 2012
@@ -150,4 +150,17 @@ public class AssociationEnhancement impl
return null;
}
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ return (o instanceof AssociationEnhancement);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
}
Modified: lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (original)
+++ lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java Wed Jan 18 22:28:07 2012
@@ -53,7 +53,7 @@ public class PayloadIterator {
this.buffer = buffer;
// TODO (Facet): avoid Multi*?
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
- this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes());
+ this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes(), false);
}
/**
Modified: lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java (original)
+++ lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java Wed Jan 18 22:28:07 2012
@@ -104,7 +104,8 @@ class ParentArray {
// TODO (Facet): avoid Multi*?
Bits liveDocs = MultiFields.getLiveDocs(indexReader);
DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(indexReader, liveDocs,
- Consts.FIELD_PAYLOADS, new BytesRef(Consts.PAYLOAD_PARENT));
+ Consts.FIELD_PAYLOADS, new BytesRef(Consts.PAYLOAD_PARENT),
+ false);
if ((positions == null || positions.advance(first) == DocsAndPositionsEnum.NO_MORE_DOCS) && first < num) {
throw new CorruptIndexException("Missing parent data for category " + first);
}
Modified: lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Wed Jan 18 22:28:07 2012
@@ -211,7 +211,7 @@ public class AllGroupHeadsCollectorTest
doc.add(group);
DocValuesField valuesField = null;
if (canUseIDV) {
- valuesField = new DocValuesField("group");
+ valuesField = new DocValuesField("group", new BytesRef(), valueType);
doc.add(valuesField);
}
Field sort1 = newField("sort1", "", StringField.TYPE_UNSTORED);
@@ -226,7 +226,7 @@ public class AllGroupHeadsCollectorTest
Field content = newField("content", "", TextField.TYPE_UNSTORED);
doc.add(content);
docNoGroup.add(content);
- NumericField id = new NumericField("id");
+ NumericField id = new NumericField("id", 0);
doc.add(id);
docNoGroup.add(id);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
@@ -257,14 +257,14 @@ public class AllGroupHeadsCollectorTest
if (groupDoc.group != null) {
group.setValue(groupDoc.group.utf8ToString());
if (canUseIDV) {
- valuesField.setBytes(new BytesRef(groupDoc.group.utf8ToString()), valueType);
+ valuesField.setValue(new BytesRef(groupDoc.group.utf8ToString()));
}
}
sort1.setValue(groupDoc.sort1.utf8ToString());
sort2.setValue(groupDoc.sort2.utf8ToString());
sort3.setValue(groupDoc.sort3.utf8ToString());
content.setValue(groupDoc.content);
- id.setIntValue(groupDoc.id);
+ id.setValue(groupDoc.id);
if (groupDoc.group == null) {
w.addDocument(docNoGroup);
} else {
@@ -276,7 +276,7 @@ public class AllGroupHeadsCollectorTest
w.close();
// NOTE: intentional but temporary field cache insanity!
- final int[] docIdToFieldId = FieldCache.DEFAULT.getInts(r, "id", false);
+ final int[] docIdToFieldId = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(r), "id", false);
final int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < docIdToFieldId.length; i++) {
int fieldId = docIdToFieldId[i];
@@ -527,9 +527,7 @@ public class AllGroupHeadsCollectorTest
private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV, Type valueType) {
doc.add(new Field(groupField, value, TextField.TYPE_STORED));
if (canUseIDV) {
- DocValuesField valuesField = new DocValuesField(groupField);
- valuesField.setBytes(new BytesRef(value), valueType);
- doc.add(valuesField);
+ doc.add(new DocValuesField(groupField, new BytesRef(value), valueType));
}
}
Modified: lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java (original)
+++ lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java Wed Jan 18 22:28:07 2012
@@ -123,9 +123,7 @@ public class AllGroupsCollectorTest exte
private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) {
doc.add(new Field(groupField, value, TextField.TYPE_STORED));
if (canUseIDV) {
- DocValuesField valuesField = new DocValuesField(groupField);
- valuesField.setBytes(new BytesRef(value), Type.BYTES_VAR_SORTED);
- doc.add(valuesField);
+ doc.add(new DocValuesField(groupField, new BytesRef(value), Type.BYTES_VAR_SORTED));
}
}
Modified: lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/solrcloud/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Wed Jan 18 22:28:07 2012
@@ -171,9 +171,7 @@ public class TestGrouping extends Lucene
private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) {
doc.add(new Field(groupField, value, TextField.TYPE_STORED));
if (canUseIDV) {
- DocValuesField valuesField = new DocValuesField(groupField);
- valuesField.setBytes(new BytesRef(value), Type.BYTES_VAR_SORTED);
- doc.add(valuesField);
+ doc.add(new DocValuesField(groupField, new BytesRef(value), Type.BYTES_VAR_SORTED));
}
}
@@ -593,7 +591,7 @@ public class TestGrouping extends Lucene
}
doc.add(newField("sort1", groupValue.sort1.utf8ToString(), StringField.TYPE_UNSTORED));
doc.add(newField("sort2", groupValue.sort2.utf8ToString(), StringField.TYPE_UNSTORED));
- doc.add(new NumericField("id").setIntValue(groupValue.id));
+ doc.add(new NumericField("id", groupValue.id));
doc.add(newField("content", groupValue.content, TextField.TYPE_UNSTORED));
//System.out.println("TEST: doc content=" + groupValue.content + " group=" + (groupValue.group == null ? "null" : groupValue.group.utf8ToString()) + " sort1=" + groupValue.sort1.utf8ToString() + " id=" + groupValue.id);
}
@@ -705,7 +703,7 @@ public class TestGrouping extends Lucene
Document doc = new Document();
Document docNoGroup = new Document();
- DocValuesField idvGroupField = new DocValuesField("group");
+ DocValuesField idvGroupField = new DocValuesField("group", new BytesRef(), Type.BYTES_VAR_SORTED);
if (canUseIDV) {
doc.add(idvGroupField);
}
@@ -721,7 +719,7 @@ public class TestGrouping extends Lucene
Field content = newField("content", "", TextField.TYPE_UNSTORED);
doc.add(content);
docNoGroup.add(content);
- NumericField id = new NumericField("id");
+ NumericField id = new NumericField("id", 0);
doc.add(id);
docNoGroup.add(id);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
@@ -747,13 +745,13 @@ public class TestGrouping extends Lucene
if (groupDoc.group != null) {
group.setValue(groupDoc.group.utf8ToString());
if (canUseIDV) {
- idvGroupField.setBytes(BytesRef.deepCopyOf(groupDoc.group), Type.BYTES_VAR_SORTED);
+ idvGroupField.setValue(BytesRef.deepCopyOf(groupDoc.group));
}
}
sort1.setValue(groupDoc.sort1.utf8ToString());
sort2.setValue(groupDoc.sort2.utf8ToString());
content.setValue(groupDoc.content);
- id.setIntValue(groupDoc.id);
+ id.setValue(groupDoc.id);
if (groupDoc.group == null) {
w.addDocument(docNoGroup);
} else {
@@ -768,7 +766,7 @@ public class TestGrouping extends Lucene
w.close();
// NOTE: intentional but temporary field cache insanity!
- final int[] docIDToID = FieldCache.DEFAULT.getInts(r, "id", false);
+ final int[] docIDToID = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(r), "id", false);
IndexReader rBlocks = null;
Directory dirBlocks = null;
@@ -801,7 +799,7 @@ public class TestGrouping extends Lucene
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
- final int[] docIDToIDBlocks = FieldCache.DEFAULT.getInts(rBlocks, "id", false);
+ final int[] docIDToIDBlocks = FieldCache.DEFAULT.getInts(new SlowMultiReaderWrapper(rBlocks), "id", false);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);
Modified: lucene/dev/branches/solrcloud/modules/join/src/java/org/apache/lucene/search/join/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/join/src/java/org/apache/lucene/search/join/package.html?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/join/src/java/org/apache/lucene/search/join/package.html (original)
+++ lucene/dev/branches/solrcloud/modules/join/src/java/org/apache/lucene/search/join/package.html Wed Jan 18 22:28:07 2012
@@ -1,7 +1,11 @@
<html>
<body>
-<p>This module supports index-time joins while searching, where joined
+<p>This modules support index-time and query-time joins.</p>
+
+<h2>Index-time joins</h2>
+
+<p>The index-time joining support joins while searching, where joined
documents are indexed as a single document block using
{@link org.apache.lucene.index.IndexWriter#addDocuments}. This is useful for any normalized content (XML documents or database tables). In database terms, all rows for all
joined tables matching a single row of the primary table must be
@@ -14,19 +18,57 @@
parent documents, as Lucene does not currently record any information
about doc blocks.</p>
-<p>At search time, use {@link org.apache.lucene.search.join.BlockJoinQuery} to remap
- matches from any child {@link org.apache.lucene.search.Query} (ie, a query that matches only
- child documents) up to the parent document space. The resulting
- {@link org.apache.lucene.search.join.BlockJoinQuery} can then be used as a clause in any query that
- matches parent documents.</p>
+<p>At search time, use {@link
+ org.apache.lucene.search.join.ToParentBlockJoinQuery} to remap/join
+ matches from any child {@link org.apache.lucene.search.Query} (ie, a
+ query that matches only child documents) up to the parent document
+ space. The
+ resulting query can then be used as a clause in any query that
+ matches parent.</p>
<p>If you only care about the parent documents matching the query, you
can use any collector to collect the parent hits, but if you'd also
like to see which child documents match for each parent document,
- use the {@link org.apache.lucene.search.join.BlockJoinCollector} to collect the hits. Once the
+ use the {@link org.apache.lucene.search.join.ToParentBlockJoinCollector} to collect the hits. Once the
search is done, you retrieve a {@link
org.apache.lucene.search.grouping.TopGroups} instance from the
- {@link org.apache.lucene.search.join.BlockJoinCollector#getTopGroups} method.</p>
+ {@link org.apache.lucene.search.join.ToParentBlockJoinCollector#getTopGroups} method.</p>
+
+<p>To map/join in the opposite direction, use {@link
+ org.apache.lucene.search.join.ToChildBlockJoinQuery}. This wraps
+ any query matching parent documents, creating the joined query
+ matching only child documents.
+
+<h2>Search-time joins</h2>
+
+<p>
+ The query time joining is terms based and implemented as two pass search. The first pass collects all the terms from a fromField
+ that match the fromQuery. The second pass returns all documents that have matching terms in a toField to the terms
+ collected in the first pass.
+</p>
+<p>Query time joining has the following input:</p>
+<ul>
+ <li><code>fromField</code>: The from field to join from.
+ <li><code>fromQuery</code>: The query executed to collect the from terms. This is usually the user specified query.
+ <li><code>multipleValuesPerDocument</code>: Whether the fromField contains more than one value per document
+ <li><code>toField</code>: The to field to join to
+</ul>
+<p>
+ Basically the query-time joining is accessible from one static method. The user of this method supplies the method
+ with the described input and a <code>IndexSearcher</code> where the from terms need to be collected from. The returned
+ query can be executed with the same <code>IndexSearcher</code>, but also with another <code>IndexSearcher</code>.
+ Example usage of the {@link org.apache.lucene.search.join.JoinUtil#createJoinQuery(String, boolean, String, org.apache.lucene.search.Query, org.apache.lucene.search.IndexSearcher)} :
+</p>
+<pre class="prettyprint">
+ String fromField = "from"; // Name of the from field
+ boolean multipleValuesPerDocument = false; // Set only yo true in the case when your fromField has multiple values per document in your index
+ String fromField = "to"; // Name of the to field
+ Query fromQuery = new TermQuery(new Term("content", searchTerm)); // Query executed to collect from values to join to the to values
+
+ MultiTermQuery joinQuery = JoinUtil.createJoinQuery(fromField, multipleValuesPerDocument, toField, fromQuery, fromSearcher);
+ TopDocs topDocs = toSearcher.search(joinQuery, 10); // Note: toSearcher can be the same as the fromSearcher
+ // Render topDocs...
+</pre>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/modules/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/branches/solrcloud/modules/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Wed Jan 18 22:28:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -25,7 +26,9 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
@@ -34,10 +37,9 @@ import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.grouping.GroupDocs;
import org.apache.lucene.search.grouping.TopGroups;
-import org.apache.lucene.search.join.BlockJoinCollector;
-import org.apache.lucene.search.join.BlockJoinQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util._TestUtil;
@@ -57,7 +59,8 @@ public class TestBlockJoin extends Lucen
private Document makeJob(String skill, int year) {
Document job = new Document();
job.add(newField("skill", skill, StringField.TYPE_STORED));
- job.add(new NumericField("year").setIntValue(year));
+ job.add(new NumericField("year", year));
+ job.add(new StoredField("year", year));
return job;
}
@@ -65,7 +68,7 @@ public class TestBlockJoin extends Lucen
private Document makeQualification(String qualification, int year) {
Document job = new Document();
job.add(newField("qualification", qualification, StringField.TYPE_STORED));
- job.add(new NumericField("year").setIntValue(year));
+ job.add(new NumericField("year", year));
return job;
}
@@ -104,14 +107,14 @@ public class TestBlockJoin extends Lucen
// Wrap the child document query to 'join' any matches
// up to corresponding parent:
- BlockJoinQuery childJoinQuery = new BlockJoinQuery(childQuery, parentsFilter, BlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ToParentBlockJoinQuery.ScoreMode.Avg);
// Combine the parent and nested child queries into a single query for a candidate
BooleanQuery fullQuery = new BooleanQuery();
fullQuery.add(new BooleanClause(parentQuery, Occur.MUST));
fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
- BlockJoinCollector c = new BlockJoinCollector(Sort.RELEVANCE, 1, true, false);
+ ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, false);
s.search(fullQuery, c);
@@ -131,9 +134,34 @@ public class TestBlockJoin extends Lucen
Document parentDoc = s.doc(group.groupValue);
assertEquals("Lisa", parentDoc.get("name"));
+
+ //System.out.println("TEST: now test up");
+
+ // Now join "up" (map parent hits to child docs) instead...:
+ ToChildBlockJoinQuery parentJoinQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter, random.nextBoolean());
+ BooleanQuery fullChildQuery = new BooleanQuery();
+ fullChildQuery.add(new BooleanClause(parentJoinQuery, Occur.MUST));
+ fullChildQuery.add(new BooleanClause(childQuery, Occur.MUST));
+
+ //System.out.println("FULL: " + fullChildQuery);
+ TopDocs hits = s.search(fullChildQuery, 10);
+ assertEquals(1, hits.totalHits);
+ childDoc = s.doc(hits.scoreDocs[0].doc);
+ //System.out.println("CHILD = " + childDoc + " docID=" + hits.scoreDocs[0].doc);
+ assertEquals("java", childDoc.get("skill"));
+ assertEquals(2007, ((StoredField) childDoc.getField("year")).numericValue());
+ assertEquals("Lisa", getParentDoc(r, parentsFilter, hits.scoreDocs[0].doc).get("name"));
r.close();
dir.close();
}
+
+ private Document getParentDoc(IndexReader reader, Filter parents, int childDocID) throws IOException {
+ final AtomicReaderContext[] leaves = ReaderUtil.leaves(reader.getTopReaderContext());
+ final int subIndex = ReaderUtil.subIndex(childDocID, leaves);
+ final AtomicReaderContext leaf = leaves[subIndex];
+ final FixedBitSet bits = (FixedBitSet) parents.getDocIdSet(leaf, null);
+ return leaf.reader.document(bits.nextSetBit(childDocID - leaf.docBase));
+ }
public void testBoostBug() throws Exception {
final Directory dir = newDirectory();
@@ -142,7 +170,7 @@ public class TestBlockJoin extends Lucen
w.close();
IndexSearcher s = newSearcher(r);
- BlockJoinQuery q = new BlockJoinQuery(new MatchAllDocsQuery(), new QueryWrapperFilter(new MatchAllDocsQuery()), BlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(new MatchAllDocsQuery(), new QueryWrapperFilter(new MatchAllDocsQuery()), ToParentBlockJoinQuery.ScoreMode.Avg);
s.search(q, 10);
BooleanQuery bq = new BooleanQuery();
bq.setBoost(2f); // we boost the BQ
@@ -199,8 +227,9 @@ public class TestBlockJoin extends Lucen
public void testRandom() throws Exception {
// We build two indices at once: one normalized (which
- // BlockJoinQuery/Collector can query) and the other w/
- // same docs just fully denormalized:
+ // ToParentBlockJoinQuery/Collector,
+ // ToChildBlockJoinQuery can query) and the other w/
+ // the same docs, just fully denormalized:
final Directory dir = newDirectory();
final Directory joinDir = newDirectory();
@@ -212,7 +241,7 @@ public class TestBlockJoin extends Lucen
// Values for child fields:
final String[][] childFields = getRandomFields(numParentDocs);
- // TODO: test star join, nested join cases too!
+ // TODO: parallel star join, nested join cases too!
final RandomIndexWriter w = new RandomIndexWriter(random, dir);
final RandomIndexWriter joinW = new RandomIndexWriter(random, joinDir);
for(int parentDocID=0;parentDocID<numParentDocs;parentDocID++) {
@@ -235,7 +264,15 @@ public class TestBlockJoin extends Lucen
final List<Document> joinDocs = new ArrayList<Document>();
if (VERBOSE) {
- System.out.println(" " + parentDoc);
+ StringBuilder sb = new StringBuilder();
+ sb.append("parentID=" + parentDoc.get("parentID"));
+ for(int fieldID=0;fieldID<parentFields.length;fieldID++) {
+ String s = parentDoc.get("parent" + fieldID);
+ if (s != null) {
+ sb.append(" parent" + fieldID + "=" + s);
+ }
+ }
+ System.out.println(" " + sb.toString());
}
final int numChildDocs = _TestUtil.nextInt(random, 1, 20);
@@ -260,7 +297,15 @@ public class TestBlockJoin extends Lucen
}
if (VERBOSE) {
- System.out.println(" " + joinChildDoc);
+ StringBuilder sb = new StringBuilder();
+ sb.append("childID=" + joinChildDoc.get("childID"));
+ for(int fieldID=0;fieldID<childFields.length;fieldID++) {
+ String s = joinChildDoc.get("child" + fieldID);
+ if (s != null) {
+ sb.append(" child" + fieldID + "=" + s);
+ }
+ }
+ System.out.println(" " + sb.toString());
}
w.addDocument(childDoc);
@@ -335,14 +380,26 @@ public class TestBlockJoin extends Lucen
random.nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT);
}
- final BlockJoinQuery childJoinQuery = new BlockJoinQuery(childQuery, parentsFilter, BlockJoinQuery.ScoreMode.Avg);
+ final int x = random.nextInt(4);
+ final ToParentBlockJoinQuery.ScoreMode agg;
+ if (x == 0) {
+ agg = ToParentBlockJoinQuery.ScoreMode.None;
+ } else if (x == 1) {
+ agg = ToParentBlockJoinQuery.ScoreMode.Max;
+ } else if (x == 2) {
+ agg = ToParentBlockJoinQuery.ScoreMode.Total;
+ } else {
+ agg = ToParentBlockJoinQuery.ScoreMode.Avg;
+ }
+
+ final ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, agg);
// To run against the block-join index:
final Query parentJoinQuery;
// Same query as parentJoinQuery, but to run against
- // the fully denormalized index (so we can compare)
- // results:
+ // the fully denormalized index (so we can compare
+ // results):
final Query parentQuery;
if (random.nextBoolean()) {
@@ -383,7 +440,7 @@ public class TestBlockJoin extends Lucen
System.out.println("\nTEST: query=" + parentQuery + " joinQuery=" + parentJoinQuery + " parentSort=" + parentSort + " childSort=" + childSort);
}
- // Merge both sorst:
+ // Merge both sorts:
final List<SortField> sortFields = new ArrayList<SortField>(Arrays.asList(parentSort.getSort()));
sortFields.addAll(Arrays.asList(childSort.getSort()));
final Sort parentAndChildSort = new Sort(sortFields.toArray(new SortField[sortFields.size()]));
@@ -412,8 +469,17 @@ public class TestBlockJoin extends Lucen
}
}
}
-
- final BlockJoinCollector c = new BlockJoinCollector(parentSort, 10, true, true);
+
+ final boolean trackScores;
+ final boolean trackMaxScore;
+ if (agg == ToParentBlockJoinQuery.ScoreMode.None) {
+ trackScores = false;
+ trackMaxScore = false;
+ } else {
+ trackScores = random.nextBoolean();
+ trackMaxScore = random.nextBoolean();
+ }
+ final ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(parentSort, 10, trackScores, trackMaxScore);
joinS.search(parentJoinQuery, c);
@@ -456,6 +522,124 @@ public class TestBlockJoin extends Lucen
} else {
compareHits(r, joinR, results, joinResults);
}
+
+ // Test joining in the opposite direction (parent to
+ // child):
+
+ // Get random query against parent documents:
+ final Query parentQuery2;
+ if (random.nextInt(3) == 2) {
+ final int fieldID = random.nextInt(parentFields.length);
+ parentQuery2 = new TermQuery(new Term("parent" + fieldID,
+ parentFields[fieldID][random.nextInt(parentFields[fieldID].length)]));
+ } else if (random.nextInt(3) == 2) {
+ BooleanQuery bq = new BooleanQuery();
+ parentQuery2 = bq;
+ final int numClauses = _TestUtil.nextInt(random, 2, 4);
+ boolean didMust = false;
+ for(int clauseIDX=0;clauseIDX<numClauses;clauseIDX++) {
+ Query clause;
+ BooleanClause.Occur occur;
+ if (!didMust && random.nextBoolean()) {
+ occur = random.nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT;
+ clause = new TermQuery(randomParentTerm(parentFields[0]));
+ didMust = true;
+ } else {
+ occur = BooleanClause.Occur.SHOULD;
+ final int fieldID = _TestUtil.nextInt(random, 1, parentFields.length-1);
+ clause = new TermQuery(new Term("parent" + fieldID,
+ parentFields[fieldID][random.nextInt(parentFields[fieldID].length)]));
+ }
+ bq.add(clause, occur);
+ }
+ } else {
+ BooleanQuery bq = new BooleanQuery();
+ parentQuery2 = bq;
+
+ bq.add(new TermQuery(randomParentTerm(parentFields[0])),
+ BooleanClause.Occur.MUST);
+ final int fieldID = _TestUtil.nextInt(random, 1, parentFields.length-1);
+ bq.add(new TermQuery(new Term("parent" + fieldID, parentFields[fieldID][random.nextInt(parentFields[fieldID].length)])),
+ random.nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT);
+ }
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: top down: parentQuery2=" + parentQuery2);
+ }
+
+ // Maps parent query to child docs:
+ final ToChildBlockJoinQuery parentJoinQuery2 = new ToChildBlockJoinQuery(parentQuery2, parentsFilter, random.nextBoolean());
+
+ // To run against the block-join index:
+ final Query childJoinQuery2;
+
+ // Same query as parentJoinQuery, but to run against
+ // the fully denormalized index (so we can compare
+ // results):
+ final Query childQuery2;
+
+ if (random.nextBoolean()) {
+ childQuery2 = parentQuery2;
+ childJoinQuery2 = parentJoinQuery2;
+ } else {
+ // AND child field w/ parent query:
+ final BooleanQuery bq = new BooleanQuery();
+ childJoinQuery2 = bq;
+ final Term childTerm = randomChildTerm(childFields[0]);
+ if (random.nextBoolean()) {
+ bq.add(parentJoinQuery2, BooleanClause.Occur.MUST);
+ bq.add(new TermQuery(childTerm),
+ BooleanClause.Occur.MUST);
+ } else {
+ bq.add(new TermQuery(childTerm),
+ BooleanClause.Occur.MUST);
+ bq.add(parentJoinQuery2, BooleanClause.Occur.MUST);
+ }
+
+ final BooleanQuery bq2 = new BooleanQuery();
+ childQuery2 = bq2;
+ if (random.nextBoolean()) {
+ bq2.add(parentQuery2, BooleanClause.Occur.MUST);
+ bq2.add(new TermQuery(childTerm),
+ BooleanClause.Occur.MUST);
+ } else {
+ bq2.add(new TermQuery(childTerm),
+ BooleanClause.Occur.MUST);
+ bq2.add(parentQuery2, BooleanClause.Occur.MUST);
+ }
+ }
+
+ final Sort childSort2 = getRandomSort("child", childFields.length);
+
+ // Search denormalized index:
+ if (VERBOSE) {
+ System.out.println("TEST: run top down query=" + childQuery2 + " sort=" + childSort2);
+ }
+ final TopDocs results2 = s.search(childQuery2, null, r.numDocs(),
+ childSort2);
+ if (VERBOSE) {
+ System.out.println(" " + results2.totalHits + " totalHits:");
+ for(ScoreDoc sd : results2.scoreDocs) {
+ final Document doc = s.doc(sd.doc);
+ System.out.println(" childID=" + doc.get("childID") + " parentID=" + doc.get("parentID") + " docID=" + sd.doc);
+ }
+ }
+
+ // Search join index:
+ if (VERBOSE) {
+ System.out.println("TEST: run top down join query=" + childJoinQuery2 + " sort=" + childSort2);
+ }
+ TopDocs joinResults2 = joinS.search(childJoinQuery2, null, joinR.numDocs(), childSort2);
+ if (VERBOSE) {
+ System.out.println(" " + joinResults2.totalHits + " totalHits:");
+ for(ScoreDoc sd : joinResults2.scoreDocs) {
+ final Document doc = joinS.doc(sd.doc);
+ final Document parentDoc = getParentDoc(joinR, parentsFilter, sd.doc);
+ System.out.println(" childID=" + doc.get("childID") + " parentID=" + parentDoc.get("parentID") + " docID=" + sd.doc);
+ }
+ }
+
+ compareChildHits(r, joinR, results2, joinResults2);
}
r.close();
@@ -464,6 +648,28 @@ public class TestBlockJoin extends Lucen
joinDir.close();
}
+ private void compareChildHits(IndexReader r, IndexReader joinR, TopDocs results, TopDocs joinResults) throws Exception {
+ assertEquals(results.totalHits, joinResults.totalHits);
+ assertEquals(results.scoreDocs.length, joinResults.scoreDocs.length);
+ for(int hitCount=0;hitCount<results.scoreDocs.length;hitCount++) {
+ ScoreDoc hit = results.scoreDocs[hitCount];
+ ScoreDoc joinHit = joinResults.scoreDocs[hitCount];
+ Document doc1 = r.document(hit.doc);
+ Document doc2 = joinR.document(joinHit.doc);
+ assertEquals("hit " + hitCount + " differs",
+ doc1.get("childID"), doc2.get("childID"));
+ // don't compare scores -- they are expected to differ
+
+
+ assertTrue(hit instanceof FieldDoc);
+ assertTrue(joinHit instanceof FieldDoc);
+
+ FieldDoc hit0 = (FieldDoc) hit;
+ FieldDoc joinHit0 = (FieldDoc) joinHit;
+ assertEquals(hit0.fields, joinHit0.fields);
+ }
+ }
+
private void compareHits(IndexReader r, IndexReader joinR, TopDocs results, TopGroups<Integer> joinResults) throws Exception {
// results is 'complete'; joinResults is a subset
int resultUpto = 0;
@@ -539,8 +745,8 @@ public class TestBlockJoin extends Lucen
// Wrap the child document query to 'join' any matches
// up to corresponding parent:
- BlockJoinQuery childJobJoinQuery = new BlockJoinQuery(childJobQuery, parentsFilter, BlockJoinQuery.ScoreMode.Avg);
- BlockJoinQuery childQualificationJoinQuery = new BlockJoinQuery(childQualificationQuery, parentsFilter, BlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery childJobJoinQuery = new ToParentBlockJoinQuery(childJobQuery, parentsFilter, ToParentBlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery childQualificationJoinQuery = new ToParentBlockJoinQuery(childQualificationQuery, parentsFilter, ToParentBlockJoinQuery.ScoreMode.Avg);
// Combine the parent and nested child queries into a single query for a candidate
BooleanQuery fullQuery = new BooleanQuery();
@@ -548,12 +754,13 @@ public class TestBlockJoin extends Lucen
fullQuery.add(new BooleanClause(childJobJoinQuery, Occur.MUST));
fullQuery.add(new BooleanClause(childQualificationJoinQuery, Occur.MUST));
- //????? How do I control volume of jobs vs qualifications per parent?
- BlockJoinCollector c = new BlockJoinCollector(Sort.RELEVANCE, 10, true, false);
+ // Collects all job and qualification child docs for
+ // each resume hit in the top N (sorted by score):
+ ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 10, true, false);
s.search(fullQuery, c);
- //Examine "Job" children
+ // Examine "Job" children
boolean showNullPointerIssue=true;
if (showNullPointerIssue) {
TopGroups<Integer> jobResults = c.getTopGroups(childJobJoinQuery, null, 0, 10, 0, true);
@@ -573,10 +780,9 @@ public class TestBlockJoin extends Lucen
assertEquals("Lisa", parentDoc.get("name"));
}
- //Now Examine qualification children
+ // Now Examine qualification children
TopGroups<Integer> qualificationResults = c.getTopGroups(childQualificationJoinQuery, null, 0, 10, 0, true);
- //!!!!! This next line can null pointer - but only if prior "jobs" section called first
assertEquals(1, qualificationResults.totalGroupedHitCount);
assertEquals(1, qualificationResults.groups.length);
@@ -610,7 +816,7 @@ public class TestBlockJoin extends Lucen
new QueryWrapperFilter(
new TermQuery(new Term("parent", "1"))));
- BlockJoinQuery q = new BlockJoinQuery(tq, parentFilter, BlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ToParentBlockJoinQuery.ScoreMode.Avg);
Weight weight = s.createNormalizedWeight(q);
DocIdSetIterator disi = weight.scorer(ReaderUtil.leaves(s.getIndexReader().getTopReaderContext())[0], true, true, null);
assertEquals(1, disi.advance(1));
@@ -644,7 +850,7 @@ public class TestBlockJoin extends Lucen
new QueryWrapperFilter(
new TermQuery(new Term("isparent", "yes"))));
- BlockJoinQuery q = new BlockJoinQuery(tq, parentFilter, BlockJoinQuery.ScoreMode.Avg);
+ ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ToParentBlockJoinQuery.ScoreMode.Avg);
Weight weight = s.createNormalizedWeight(q);
DocIdSetIterator disi = weight.scorer(ReaderUtil.leaves(s.getIndexReader().getTopReaderContext())[0], true, true, null);
assertEquals(2, disi.advance(0));
Modified: lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java (original)
+++ lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java Wed Jan 18 22:28:07 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.queries.functi
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
@@ -61,11 +62,12 @@ public class OrdFieldSource extends Valu
}
+ // TODO: this is trappy? perhaps this query instead should make you pass a slow reader yourself?
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final int off = readerContext.docBase;
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader;
- final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(topReader, field);
+ final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(topReader), field);
return new IntDocValues(this) {
protected String toTerm(String readableValue) {
return readableValue;