You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2010/02/26 14:10:08 UTC
svn commit: r916666 [6/16] - in /lucene/java/branches/flex_1458: ./ contrib/
contrib/analyzers/common/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/
contrib/analyzers/c...
Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Fri Feb 26 13:09:54 2010
@@ -979,6 +979,79 @@
return algLines;
}
+ /**
+ * Test that we can create ShingleAnalyzerWrappers.
+ */
+ public void testShingleAnalyzer() throws Exception {
+ String text = "one,two,three, four five six";
+
+ // Default analyzer, maxShingleSize, and outputUnigrams
+ Benchmark benchmark = execBenchmark(getShingleConfig(""));
+ TokenStream stream = benchmark.getRunData().getAnalyzer().tokenStream
+ ("bogus", new StringReader(text));
+ assertEqualShingle(benchmark.getRunData().getAnalyzer(), text,
+ new String[] {"one", "one two", "two", "two three",
+ "three", "three four", "four", "four five",
+ "five", "five six", "six"});
+ // Default analyzer, maxShingleSize = 3, and outputUnigrams = false
+ benchmark = execBenchmark
+ (getShingleConfig("maxShingleSize:3,outputUnigrams:false"));
+ assertEqualShingle(benchmark.getRunData().getAnalyzer(), text,
+ new String[] { "one two", "one two three", "two three",
+ "two three four", "three four",
+ "three four five", "four five",
+ "four five six", "five six" });
+ // WhitespaceAnalyzer, default maxShingleSize and outputUnigrams
+ benchmark = execBenchmark
+ (getShingleConfig("analyzer:WhitespaceAnalyzer"));
+ assertEqualShingle(benchmark.getRunData().getAnalyzer(), text,
+ new String[] { "one,two,three,", "one,two,three, four",
+ "four", "four five", "five", "five six",
+ "six" });
+
+ // WhitespaceAnalyzer, maxShingleSize=3 and outputUnigrams=false
+ benchmark = execBenchmark
+ (getShingleConfig
+ ("outputUnigrams:false,maxShingleSize:3,analyzer:WhitespaceAnalyzer"));
+ assertEqualShingle(benchmark.getRunData().getAnalyzer(), text,
+ new String[] { "one,two,three, four",
+ "one,two,three, four five",
+ "four five", "four five six",
+ "five six" });
+ }
+
+ private void assertEqualShingle
+ (Analyzer analyzer, String text, String[] expected) throws Exception {
+ TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text));
+ stream.reset();
+ TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+ int termNum = 0;
+ while (stream.incrementToken()) {
+ assertTrue("Extra output term(s), starting with '"
+ + new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'",
+ termNum < expected.length);
+ assertEquals("Mismatch in output term # " + termNum + " - ",
+ expected[termNum],
+ new String(termAtt.termBuffer(), 0, termAtt.termLength()));
+ ++termNum;
+ }
+ assertEquals("Too few output terms", expected.length, termNum);
+ stream.close();
+ }
+
+ private static String[] getShingleConfig(String params) {
+ String algLines[] = {
+ "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "NewShingleAnalyzer(" + params + ")",
+ "CreateIndex",
+ "{ \"AddDocs\" AddDoc > : * "
+ };
+ return algLines;
+ }
+
private static String getReuters20LinesFile() {
return System.getProperty("lucene.common.dir").replace('\\','/') +
"/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt";
Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java Fri Feb 26 13:09:54 2010
@@ -20,10 +20,11 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.PrintWriter;
import org.apache.lucene.benchmark.byTask.TestPerfTasksLogic;
-import org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource;
import org.apache.lucene.benchmark.quality.Judge;
import org.apache.lucene.benchmark.quality.QualityQuery;
import org.apache.lucene.benchmark.quality.QualityQueryParser;
@@ -39,6 +40,10 @@
/**
* Test that quality run does its job.
+ * <p>
+ * NOTE: if the default scoring or StandardAnalyzer is changed, then
+ * this test will not work correctly, as it does not dynamically
+ * generate its test trec topics/qrels!
*/
public class TestQualityRun extends TestCase {
@@ -52,14 +57,14 @@
}
public void testTrecQuality() throws Exception {
- // first create the complete reuters index
+ // first create the partial reuters index
createReutersIndex();
File workDir = new File(System.getProperty("benchmark.work.dir","work"));
assertTrue("Bad workDir: "+workDir, workDir.exists()&& workDir.isDirectory());
int maxResults = 1000;
- String docNameField = "docid";
+ String docNameField = "doctitle"; // orig docID is in the linedoc format title
PrintWriter logger = DEBUG ? new PrintWriter(System.out,true) : null;
@@ -105,13 +110,13 @@
assertTrue("avg-p should be hurt: "+s.getAvp(), 1.0 > s.getAvp());
assertTrue("recall should be hurt: "+s.getRecall(), 1.0 > s.getRecall());
for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
- assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-9);
+ assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
}
break;
case 1:
assertTrue("avg-p should be hurt", 1.0 > s.getAvp());
- assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-9);
+ assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2);
for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
assertTrue("p_at_"+j+" should be hurt: "+s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j));
}
@@ -126,10 +131,10 @@
break;
default: {
- assertEquals("avg-p should be perfect: "+s.getAvp(), 1.0, s.getAvp(), 1E-9);
- assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-9);
+ assertEquals("avg-p should be perfect: "+s.getAvp(), 1.0, s.getAvp(), 1E-2);
+ assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2);
for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
- assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-9);
+ assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
}
}
@@ -149,13 +154,45 @@
}
+
+ public void testTrecTopicsReader() throws Exception {
+ // prepare topics
+ InputStream topicsFile = getClass().getResourceAsStream("trecTopics.txt");
+ TrecTopicsReader qReader = new TrecTopicsReader();
+ QualityQuery qqs[] = qReader.readQueries(
+ new BufferedReader(new InputStreamReader(topicsFile, "UTF-8")));
+
+ assertEquals(20, qqs.length);
+
+ QualityQuery qq = qqs[0];
+ assertEquals("statement months total 1987", qq.getValue("title"));
+ assertEquals("Topic 0 Description Line 1 Topic 0 Description Line 2",
+ qq.getValue("description"));
+ assertEquals("Topic 0 Narrative Line 1 Topic 0 Narrative Line 2",
+ qq.getValue("narrative"));
+
+ qq = qqs[1];
+ assertEquals("agreed 15 against five", qq.getValue("title"));
+ assertEquals("Topic 1 Description Line 1 Topic 1 Description Line 2",
+ qq.getValue("description"));
+ assertEquals("Topic 1 Narrative Line 1 Topic 1 Narrative Line 2",
+ qq.getValue("narrative"));
+
+ qq = qqs[19];
+ assertEquals("20 while common week", qq.getValue("title"));
+ assertEquals("Topic 19 Description Line 1 Topic 19 Description Line 2",
+ qq.getValue("description"));
+ assertEquals("Topic 19 Narrative Line 1 Topic 19 Narrative Line 2",
+ qq.getValue("narrative"));
+ }
- // use benchmark logic to create the full Reuters index
+ // use benchmark logic to create the mini Reuters index
private void createReutersIndex() throws Exception {
// 1. alg definition
String algLines[] = {
"# ----- properties ",
- "content.source="+ReutersContentSource.class.getName(),
+ "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
+ "docs.file=" + getReuters578LinesFile(),
"content.source.log.step=2500",
"doc.term.vector=false",
"content.source.forever=false",
@@ -172,4 +209,9 @@
// 2. execute the algorithm (required in every "logic" test)
TestPerfTasksLogic.execBenchmark(algLines);
}
+
+ private static String getReuters578LinesFile() {
+ return System.getProperty("lucene.common.dir").replace('\\','/') +
+ "/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/reuters.578.lines.txt.bz2";
+ }
}
Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt Fri Feb 26 13:09:54 2010
@@ -26,10 +26,12 @@
<title> statement months total 1987
<desc> Description:
-
+Topic 0 Description Line 1
+Topic 0 Description Line 2
<narr> Narrative:
-
+Topic 0 Narrative Line 1
+Topic 0 Narrative Line 2
</top>
@@ -39,10 +41,12 @@
<title> agreed 15 against five
<desc> Description:
-
+Topic 1 Description Line 1
+Topic 1 Description Line 2
<narr> Narrative:
-
+Topic 1 Narrative Line 1
+Topic 1 Narrative Line 2
</top>
@@ -273,9 +277,11 @@
<title> 20 while common week
<desc> Description:
-
+Topic 19 Description Line 1
+Topic 19 Description Line 2
<narr> Narrative:
-
+Topic 19 Narrative Line 1
+Topic 19 Narrative Line 2
</top>
Modified: lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml (original)
+++ lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml Fri Feb 26 13:09:54 2010
@@ -21,10 +21,10 @@
Lucene Berkeley DB Java Edition integration
</description>
- <property name="je.version" value="3.3.69" />
+ <property name="je.version" value="3.3.93" />
<path id="je.jar">
- <pathelement location="lib/je-${je.version}/lib/je-${je.version}.jar" />
+ <pathelement location="lib/je-${je.version}.jar" />
</path>
<available classname="com.sleepycat.je.Database" property="je.jar.exists">
@@ -48,13 +48,8 @@
<target name="get-je-jar" unless="je.jar.exists">
<mkdir dir="lib" />
- <get src="http://download.oracle.com/berkeley-db/je-${je.version}.zip"
- dest="lib/je-${je.version}.zip" />
- <unzip src="lib/je-${je.version}.zip" dest="lib">
- <patternset>
- <include name="je-${je.version}/lib/je-${je.version}.jar" />
- </patternset>
- </unzip>
+ <get src="http://download.oracle.com/maven/com/sleepycat/je/${je.version}/je-${je.version}.jar"
+ dest="lib/je-${je.version}.jar" />
</target>
<target name="check-and-get-je-jar" depends="get-je-jar" />
Modified: lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template (original)
+++ lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template Fri Feb 26 13:09:54 2010
@@ -36,7 +36,7 @@
<dependency>
<groupId>sleepycat</groupId>
<artifactId>je</artifactId>
- <version>1.7.0</version>
+ <version>${sleepycat-je-version}</version>
</dependency>
</dependencies>
</project>
Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Fri Feb 26 13:09:54 2010
@@ -22,6 +22,7 @@
import java.util.List;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.MapFieldSelector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
@@ -72,7 +73,7 @@
List<WeightedFragInfo> fragInfos = getWeightedFragInfoList( fieldFragList.fragInfos );
List<String> fragments = new ArrayList<String>( maxNumFragments );
- String[] values = getFieldValues( reader, docId, fieldName );
+ Field[] values = getFields( reader, docId, fieldName );
if( values.length == 0 ) return null;
StringBuilder buffer = new StringBuilder();
int[] nextValueIndex = { 0 };
@@ -83,15 +84,31 @@
return fragments.toArray( new String[fragments.size()] );
}
+ @Deprecated
protected String[] getFieldValues( IndexReader reader, int docId, String fieldName) throws IOException {
Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) );
return doc.getValues( fieldName ); // according to Document class javadoc, this never returns null
}
+
+ protected Field[] getFields( IndexReader reader, int docId, String fieldName) throws IOException {
+ // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field???
+ Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) );
+ return doc.getFields( fieldName ); // according to Document class javadoc, this never returns null
+ }
+ @Deprecated
protected String makeFragment( StringBuilder buffer, int[] index, String[] values, WeightedFragInfo fragInfo ){
- StringBuilder fragment = new StringBuilder();
final int s = fragInfo.startOffset;
- String src = getFragmentSource( buffer, index, values, s, fragInfo.endOffset );
+ return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s );
+ }
+
+ protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo ){
+ final int s = fragInfo.startOffset;
+ return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s );
+ }
+
+ private String makeFragment( WeightedFragInfo fragInfo, String src, int s ){
+ StringBuilder fragment = new StringBuilder();
int srcIndex = 0;
for( SubInfo subInfo : fragInfo.subInfos ){
for( Toffs to : subInfo.termsOffsets ){
@@ -104,6 +121,7 @@
return fragment.toString();
}
+ @Deprecated
protected String getFragmentSource( StringBuilder buffer, int[] index, String[] values,
int startOffset, int endOffset ){
while( buffer.length() < endOffset && index[0] < values.length ){
@@ -114,6 +132,17 @@
int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
return buffer.substring( startOffset, eo );
}
+
+ protected String getFragmentSource( StringBuilder buffer, int[] index, Field[] values,
+ int startOffset, int endOffset ){
+ while( buffer.length() < endOffset && index[0] < values.length ){
+ if( index[0] > 0 && values[index[0]].isTokenized() && values[index[0]].stringValue().length() > 0 )
+ buffer.append( ' ' );
+ buffer.append( values[index[0]++].stringValue() );
+ }
+ int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
+ return buffer.substring( startOffset, eo );
+ }
protected String getPreTag( int num ){
return preTags.length > num ? preTags[num] : preTags[0];
Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Fri Feb 26 13:09:54 2010
@@ -27,6 +27,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -78,6 +79,12 @@
flatten( clause.getQuery(), flatQueries );
}
}
+ else if( sourceQuery instanceof DisjunctionMaxQuery ){
+ DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery;
+ for( Query query : dmq ){
+ flatten( query, flatQueries );
+ }
+ }
else if( sourceQuery instanceof TermQuery ){
if( !flatQueries.contains( sourceQuery ) )
flatQueries.add( sourceQuery );
Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Fri Feb 26 13:09:54 2010
@@ -24,6 +24,7 @@
import junit.framework.TestCase;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -40,6 +41,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -55,6 +57,7 @@
protected Directory dir;
protected Analyzer analyzerW;
protected Analyzer analyzerB;
+ protected Analyzer analyzerK;
protected IndexReader reader;
protected QueryParser paW;
protected QueryParser paB;
@@ -75,11 +78,18 @@
"\nLucene/Solr does not require such additional hardware.",
"\nWhen you talk about processing speed, the"
};
+
+ protected static final String[] strMVValues = {
+ "abc",
+ "defg",
+ "hijkl"
+ };
@Override
protected void setUp() throws Exception {
- analyzerW = new WhitespaceAnalyzer();
+ analyzerW = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
analyzerB = new BigramAnalyzer();
+ analyzerK = new KeywordAnalyzer();
paW = new QueryParser(Version.LUCENE_CURRENT, F, analyzerW );
paB = new QueryParser(Version.LUCENE_CURRENT, F, analyzerB );
dir = new RAMDirectory();
@@ -141,6 +151,18 @@
return query;
}
+ protected Query dmq( Query... queries ){
+ return dmq( 0.0F, queries );
+ }
+
+ protected Query dmq( float tieBreakerMultiplier, Query... queries ){
+ DisjunctionMaxQuery query = new DisjunctionMaxQuery( tieBreakerMultiplier );
+ for( Query q : queries ){
+ query.add( q );
+ }
+ return query;
+ }
+
protected void assertCollectionQueries( Collection<Query> actual, Query... expected ){
assertEquals( expected.length, actual.size() );
for( Query query : expected ){
@@ -205,7 +227,7 @@
public boolean incrementToken() throws IOException {
if( !getNextPartialSnippet() )
return false;
-
+ clearAttributes();
termAtt.setTermBuffer(snippet, startTerm, lenTerm);
offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
return true;
@@ -301,6 +323,7 @@
make1dmfIndex( analyzerB, values );
}
+ // make 1 doc with multi valued field
protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
IndexWriter writer = new IndexWriter( dir, analyzer, true, MaxFieldLength.LIMITED );
Document doc = new Document();
@@ -312,6 +335,18 @@
reader = IndexReader.open( dir, true );
}
+ // make 1 doc with multi valued & not analyzed field
+ protected void make1dmfIndexNA( String... values ) throws Exception {
+ IndexWriter writer = new IndexWriter( dir, analyzerK, true, MaxFieldLength.LIMITED );
+ Document doc = new Document();
+ for( String value: values )
+ doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ writer.addDocument( doc );
+ writer.close();
+
+ reader = IndexReader.open( dir, true );
+ }
+
protected void makeIndexShortMV() throws Exception {
// 012345
@@ -373,4 +408,18 @@
make1dmfIndexB( biMVValues );
}
+
+ protected void makeIndexStrMV() throws Exception {
+
+ // 0123
+ // "abc"
+
+ // 34567
+ // "defg"
+
+ // 111
+ // 789012
+ // "hijkl"
+ make1dmfIndexNA( strMVValues );
+ }
}
Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java Fri Feb 26 13:09:54 2010
@@ -38,6 +38,14 @@
assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) );
}
+ public void testFlattenDisjunctionMaxQuery() throws Exception {
+ Query query = dmq( tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
+ FieldQuery fq = new FieldQuery( query, true, true );
+ Set<Query> flatQueries = new HashSet<Query>();
+ fq.flatten( query, flatQueries );
+ assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
+ }
+
public void testFlattenTermAndPhrase() throws Exception {
Query query = paW.parse( "A AND \"B C\"" );
FieldQuery fq = new FieldQuery( query, true, true );
Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Fri Feb 26 13:09:54 2010
@@ -127,4 +127,16 @@
reader = IndexReader.open( dir, true );
}
+
+ public void test1StrMV() throws Exception {
+ makeIndexStrMV();
+
+ FieldQuery fq = new FieldQuery( tq( "defg" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ SimpleFragListBuilder sflb = new SimpleFragListBuilder();
+ FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
+ SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
+ assertEquals( "abc<b>defg</b>hijkl", sfb.createFragment( reader, 0, F, ffl ) );
+ }
}
Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Fri Feb 26 13:09:54 2010
@@ -169,6 +169,7 @@
return false;
}
Token token = tokens[currentToken++];
+ clearAttributes();
termAtt.setTermBuffer(token.term());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Fri Feb 26 13:09:54 2010
@@ -102,6 +102,7 @@
public boolean incrementToken() throws IOException {
if (this.tokensAtCurrentPosition.hasNext()) {
final Token next = this.tokensAtCurrentPosition.next();
+ clearAttributes();
termAttribute.setTermBuffer(next.term());
positionIncrementAttribute.setPositionIncrement(next
.getPositionIncrement());
Propchange: lucene/java/branches/flex_1458/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 26 13:09:54 2010
@@ -1,5 +1,5 @@
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
-/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850
+/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
/lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906
-/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889622,889667,889866-899001
+/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889614-916543
Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Fri Feb 26 13:09:54 2010
@@ -49,6 +49,7 @@
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Version;
import junit.framework.TestCase;
@@ -60,7 +61,7 @@
final String TEXT = "the fox jumped";
final Directory directory = new RAMDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED);
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED);
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@@ -103,7 +104,7 @@
final String TEXT = "the fox jumped";
final Directory directory = new RAMDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED);
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED);
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@@ -168,7 +169,7 @@
final String TEXT = "the fox did not jump";
final Directory directory = new RAMDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED);
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED);
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),
@@ -210,7 +211,7 @@
final String TEXT = "the fox did not jump";
final Directory directory = new RAMDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED);
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED);
try {
final Document document = new Document();
document.add(new Field(FIELD, TEXT, Store.YES, Index.ANALYZED,
@@ -250,7 +251,7 @@
final String TEXT = "the fox did not jump";
final Directory directory = new RAMDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED);
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED);
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),
@@ -311,6 +312,7 @@
if (this.i >= this.tokens.length) {
return false;
}
+ clearAttributes();
termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
.term().length());
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
@@ -355,6 +357,7 @@
if (this.i >= this.tokens.length) {
return false;
}
+ clearAttributes();
termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
.term().length());
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri Feb 26 13:09:54 2010
@@ -118,7 +118,7 @@
}
public void testQueryScorerHits() throws Exception {
- Analyzer analyzer = new SimpleAnalyzer();
+ Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_CURRENT);
QueryParser qp = new QueryParser(TEST_VERSION, FIELD_NAME, analyzer);
query = qp.parse("\"very long\"");
searcher = new IndexSearcher(ramDir, true);
@@ -226,7 +226,7 @@
String f2c = f2 + ":";
String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2
+ " OR " + f2c + ph2 + ")";
- Analyzer analyzer = new WhitespaceAnalyzer();
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
QueryParser qp = new QueryParser(TEST_VERSION, f1, analyzer);
Query query = qp.parse(q);
@@ -1458,6 +1458,7 @@
public boolean incrementToken() throws IOException {
if(iter.hasNext()) {
Token token = iter.next();
+ clearAttributes();
termAtt.setTermBuffer(token.term());
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -1506,6 +1507,7 @@
public boolean incrementToken() throws IOException {
if(iter.hasNext()) {
Token token = iter.next();
+ clearAttributes();
termAtt.setTermBuffer(token.term());
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -1527,64 +1529,64 @@
Highlighter highlighter;
String result;
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("foo");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("10");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("10");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("speed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hispeed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi speed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
// ///////////////// same tests, just put the bigger overlapping token
// first
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("foo");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("10");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("10");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("speed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hispeed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi speed");
+ query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
@@ -1595,7 +1597,7 @@
}
private Directory dir = new RAMDirectory();
- private Analyzer a = new WhitespaceAnalyzer();
+ private Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
public void testWeightedTermsWithDeletes() throws IOException, ParseException, InvalidTokenOffsetsException {
makeIndex();
@@ -1799,7 +1801,7 @@
*/
@Override
public TokenStream tokenStream(String arg0, Reader arg1) {
- LowerCaseTokenizer stream = new LowerCaseTokenizer(arg1);
+ LowerCaseTokenizer stream = new LowerCaseTokenizer(Version.LUCENE_CURRENT, arg1);
stream.addAttribute(TermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
@@ -1845,6 +1847,7 @@
return false;
}
//Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
+ clearAttributes();
termAtt.setTermBuffer(realTermAtt.term());
offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
@@ -1862,6 +1865,7 @@
return true;
} else {
String tok = st.nextToken();
+ clearAttributes();
termAtt.setTermBuffer(tok);
offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
posIncrAtt.setPositionIncrement(0);
Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original)
+++ lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Fri Feb 26 13:09:54 2010
@@ -43,7 +43,7 @@
* results up to a 100 times faster than the file-centric RAMDirectory
* at the cost of greater RAM consumption.
* <p>
- * WARNING: This contrib is experimental and the APIs may change without warning.
+ * @lucene.experimental
* <p>
* There are no read and write locks in this store.
* {@link InstantiatedIndexReader} {@link InstantiatedIndexReader#isCurrent()} all the time
Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html (original)
+++ lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html Fri Feb 26 13:09:54 2010
@@ -22,7 +22,7 @@
<body>
<p>InstantiatedIndex, alternative RAM store for small corpora.</p>
-<p>WARNING: This contrib is experimental and the APIs may change without warning.</p>
+<p>@lucene.experimental</p>
<h2>Abstract</h2>
<p>
Propchange: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Fri Feb 26 13:09:54 2010
@@ -0,0 +1 @@
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334
Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java (original)
+++ lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java Fri Feb 26 13:09:54 2010
@@ -20,6 +20,7 @@
import junit.framework.TestCase;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
@@ -35,7 +36,7 @@
Directory dir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
Document doc = new Document();
doc.add(new Field("foo", "bar rab abr bra rba", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
doc.add(new Field("moo", "bar rab abr bra rba", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java (original)
+++ lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java Fri Feb 26 13:09:54 2010
@@ -24,6 +24,7 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -35,17 +36,17 @@
public void test() throws Exception {
Directory dir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
addDocument(iw, "Hello, world!");
addDocument(iw, "All work and no play makes jack a dull boy");
iw.close();
- iw = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
+ iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.UNLIMITED);
addDocument(iw, "Hello, tellus!");
addDocument(iw, "All work and no play makes danny a dull boy");
iw.close();
- iw = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
+ iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.UNLIMITED);
addDocument(iw, "Hello, earth!");
addDocument(iw, "All work and no play makes wendy a dull girl");
iw.close();
Modified: lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Feb 26 13:09:54 2010
@@ -287,6 +287,7 @@
throw new IllegalArgumentException("keyword must not be null");
String term = obj.toString();
+ clearAttributes();
termAtt.setTermBuffer(term);
offsetAtt.setOffset(start, start+termAtt.termLength());
start += term.length() + 1; // separate words by 1 (blank) character
Modified: lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri Feb 26 13:09:54 2010
@@ -277,10 +277,10 @@
Set<?> stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
Analyzer[] analyzers = new Analyzer[] {
- new SimpleAnalyzer(),
+ new SimpleAnalyzer(Version.LUCENE_CURRENT),
new StopAnalyzer(Version.LUCENE_CURRENT),
new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT),
-// new WhitespaceAnalyzer(),
+// new WhitespaceAnalyzer(Version.LUCENE_CURRENT),
// new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, false, null),
// new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, true, stopWords),
// new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS),
Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Fri Feb 26 13:09:54 2010
@@ -41,8 +41,7 @@
* File{In,Out}putStream) so it will not work with non
* FSDirectory Directory impls.</p>
*
- * <p><b>NOTE</b>: The tool is experimental and might change
- * in incompatible ways in the next release. You can easily
+ * @lucene.experimental You can easily
* accidentally remove segments from your index so be
* careful!
*/
Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java Fri Feb 26 13:09:54 2010
@@ -27,7 +27,6 @@
/**
* Merges indices specified on the command line into the index
* specified as the first command line argument.
- * @version $Id$
*/
public class IndexMergeTool {
public static void main(String[] args) throws IOException {
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Fri Feb 26 13:09:54 2010
@@ -34,11 +34,10 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
/**
* Tests changing of field norms with a custom similarity and with fake norms.
- *
- * @version $Id$
*/
public class TestFieldNormModifier extends TestCase {
public TestFieldNormModifier(String name) {
@@ -59,7 +58,7 @@
@Override
public void setUp() throws Exception {
- IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true, MaxFieldLength.UNLIMITED);
+ IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED);
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Fri Feb 26 13:09:54 2010
@@ -23,6 +23,7 @@
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
public class TestIndexSplitter extends LuceneTestCase {
@@ -35,7 +36,7 @@
_TestUtil.rmDir(destDir);
destDir.mkdirs();
FSDirectory fsDir = FSDirectory.open(dir);
- IndexWriter iw = new IndexWriter(fsDir, new WhitespaceAnalyzer(), true, MaxFieldLength.UNLIMITED);
+ IndexWriter iw = new IndexWriter(fsDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED);
for (int x=0; x < 100; x++) {
Document doc = TestIndexWriterReader.createDocument(x, "index", 5);
iw.addDocument(doc);
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Fri Feb 26 13:09:54 2010
@@ -22,6 +22,7 @@
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
import junit.framework.TestCase;
@@ -32,7 +33,7 @@
@Override
public void setUp() throws Exception {
RAMDirectory dir = new RAMDirectory();
- IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true,
+ IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true,
MaxFieldLength.LIMITED);
Document doc;
for (int i = 0; i < NUM_DOCS; i++) {
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java Fri Feb 26 13:09:54 2010
@@ -43,6 +43,7 @@
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
public class ChainedFilterTest extends TestCase {
public static final int MAX = 500;
@@ -59,7 +60,7 @@
public void setUp() throws Exception {
directory = new RAMDirectory();
IndexWriter writer =
- new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ new IndexWriter(directory, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
Calendar cal = new GregorianCalendar();
cal.clear();
@@ -187,7 +188,7 @@
public void testWithCachingFilter() throws Exception {
Directory dir = new RAMDirectory();
- Analyzer analyzer = new WhitespaceAnalyzer();
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
IndexWriter writer = new IndexWriter(dir, analyzer, true, MaxFieldLength.LIMITED);
writer.close();
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Fri Feb 26 13:09:54 2010
@@ -37,11 +37,10 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
/**
* Tests changing the norms after changing the simularity
- *
- * @version $Id:$
*/
public class TestLengthNormModifier extends TestCase {
public TestLengthNormModifier(String name) {
@@ -62,7 +61,7 @@
@Override
public void setUp() throws Exception {
- IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true, MaxFieldLength.UNLIMITED);
+ IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED);
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java Fri Feb 26 13:09:54 2010
@@ -47,7 +47,7 @@
public QueryParser getParser(Analyzer a, Extensions extensions)
throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
QueryParser qp = extensions == null ? new ExtendableQueryParser(
Version.LUCENE_CURRENT, "field", a) : new ExtendableQueryParser(
Version.LUCENE_CURRENT, "field", a, extensions);
Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Fri Feb 26 13:09:54 2010
@@ -36,6 +36,7 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.LocalizedTestCase;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -73,6 +74,7 @@
@Override
public boolean incrementToken() throws IOException {
+ clearAttributes();
if (inPhrase) {
inPhrase = false;
termAtt.setTermBuffer("phrase2");
@@ -98,7 +100,7 @@
/** Filters LowerCaseTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new LowerCaseTokenizer(reader));
+ return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader));
}
}
@@ -128,7 +130,7 @@
public PrecedenceQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a);
qp.setDefaultOperator(PrecedenceQueryParser.OR_OPERATOR);
return qp;
@@ -173,7 +175,7 @@
public Query getQueryDOA(String query, Analyzer a)
throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a);
qp.setDefaultOperator(PrecedenceQueryParser.AND_OPERATOR);
return qp.parse(query);
@@ -253,7 +255,7 @@
}
public void testPunct() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -411,7 +413,7 @@
}
public void testEscaped() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
/*assertQueryEquals("\\[brackets", a, "\\[brackets");
assertQueryEquals("\\[brackets", null, "brackets");
@@ -543,7 +545,7 @@
public void testCustomQueryParserWildcard() {
try {
- new QPTestParser("contents", new WhitespaceAnalyzer()).parse("a?t");
+ new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t");
} catch (ParseException expected) {
return;
}
@@ -552,7 +554,7 @@
public void testCustomQueryParserFuzzy() throws Exception {
try {
- new QPTestParser("contents", new WhitespaceAnalyzer()).parse("xunit~");
+ new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~");
} catch (ParseException expected) {
return;
}
@@ -562,7 +564,7 @@
public void testBooleanQuery() throws Exception {
BooleanQuery.setMaxClauseCount(2);
try {
- getParser(new WhitespaceAnalyzer()).parse("one two three");
+ getParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("one two three");
fail("ParseException expected due to too many boolean clauses");
} catch (ParseException expected) {
// too many boolean clauses, so ParseException is expected
@@ -576,7 +578,7 @@
// failing tests disabled since PrecedenceQueryParser
// is currently unmaintained
public void _testPrecedence() throws Exception {
- PrecedenceQueryParser parser = getParser(new WhitespaceAnalyzer());
+ PrecedenceQueryParser parser = getParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
Query query1 = parser.parse("A AND B OR C AND D");
Query query2 = parser.parse("(A AND B) OR (C AND D)");
assertEquals(query1, query2);
Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java Fri Feb 26 13:09:54 2010
@@ -28,6 +28,7 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
public class BooleanFilterTest extends TestCase
{
@@ -38,7 +39,7 @@
protected void setUp() throws Exception
{
directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
//Add series of docs with filterable fields : acces rights, prices, dates and "in-stock" flags
addDoc(writer, "admin guest", "010", "20040101","Y");
Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java Fri Feb 26 13:09:54 2010
@@ -30,12 +30,13 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
public class FuzzyLikeThisQueryTest extends TestCase
{
private RAMDirectory directory;
private IndexSearcher searcher;
- private Analyzer analyzer=new WhitespaceAnalyzer();
+ private Analyzer analyzer=new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
@Override
protected void setUp() throws Exception
@@ -114,7 +115,7 @@
}
public void testFuzzyLikeThisQueryEquals() {
- Analyzer analyzer = new WhitespaceAnalyzer();
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
FuzzyLikeThisQuery fltq1 = new FuzzyLikeThisQuery(10, analyzer);
fltq1.addTerms("javi", "subject", 0.5f, 2);
FuzzyLikeThisQuery fltq2 = new FuzzyLikeThisQuery(10, analyzer);
Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Fri Feb 26 13:09:54 2010
@@ -30,6 +30,7 @@
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Version;
public class TermsFilterTest extends TestCase
{
@@ -55,7 +56,7 @@
{
String fieldName="field1";
RAMDirectory rd=new RAMDirectory();
- IndexWriter w=new IndexWriter(rd,new WhitespaceAnalyzer(),MaxFieldLength.UNLIMITED);
+ IndexWriter w=new IndexWriter(rd,new WhitespaceAnalyzer(Version.LUCENE_CURRENT),MaxFieldLength.UNLIMITED);
for (int i = 0; i < 100; i++)
{
Document doc=new Document();
Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java (original)
+++ lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java Fri Feb 26 13:09:54 2010
@@ -33,7 +33,7 @@
@Override
public String toString() {
- return "<matchAllDocs field='*' term='*'>";
+ return "<matchAllDocs field='*' term='*'/>";
}
public CharSequence toQueryString(EscapeQuerySyntax escapeSyntaxParser) {
Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Fri Feb 26 13:09:54 2010
@@ -119,6 +119,7 @@
public boolean incrementToken() throws IOException {
if (inPhrase) {
inPhrase = false;
+ clearAttributes();
termAtt.setTermBuffer("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
@@ -143,7 +144,7 @@
/** Filters LowerCaseTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new LowerCaseTokenizer(reader));
+ return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader));
}
}
@@ -203,7 +204,7 @@
public StandardQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
@@ -293,7 +294,7 @@
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
qp.setDefaultOperator(Operator.AND);
@@ -313,7 +314,7 @@
}
public void testConstantScoreAutoRewrite() throws Exception {
- StandardQueryParser qp = new StandardQueryParser(new WhitespaceAnalyzer());
+ StandardQueryParser qp = new StandardQueryParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
Query q = qp.parse("foo*bar", "field");
assertTrue(q instanceof WildcardQuery);
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((MultiTermQuery) q).getRewriteMethod());
@@ -338,9 +339,9 @@
public void testSimple() throws Exception {
assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
assertQueryEquals("term term term", null, "term term term");
- assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(),
+ assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(Version.LUCENE_CURRENT),
"t�rm term term");
- assertQueryEquals("�mlaut", new WhitespaceAnalyzer(), "�mlaut");
+ assertQueryEquals("�mlaut", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "�mlaut");
assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:");
@@ -397,7 +398,7 @@
}
public void testPunct() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -572,7 +573,7 @@
public void testFarsiRangeCollating() throws Exception {
RAMDirectory ramDir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true,
+ IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES,
@@ -582,7 +583,7 @@
IndexSearcher is = new IndexSearcher(ramDir, true);
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new WhitespaceAnalyzer());
+ qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
// RuleBasedCollator. However, the Arabic Locale seems to order the
@@ -736,7 +737,7 @@
}
public void testEscaped() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
/*
* assertQueryEquals("\\[brackets", a, "\\[brackets");
@@ -835,7 +836,7 @@
}
public void testQueryStringEscaping() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
@@ -950,7 +951,7 @@
public void testCustomQueryParserWildcard() {
try {
- new QPTestParser(new WhitespaceAnalyzer()).parse("a?t", "contents");
+ new QPTestParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t", "contents");
fail("Wildcard queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -959,7 +960,7 @@
public void testCustomQueryParserFuzzy() throws Exception {
try {
- new QPTestParser(new WhitespaceAnalyzer()).parse("xunit~", "contents");
+ new QPTestParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~", "contents");
fail("Fuzzy queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -970,7 +971,7 @@
BooleanQuery.setMaxClauseCount(2);
try {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new WhitespaceAnalyzer());
+ qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
qp.parse("one two three", "field");
fail("ParseException expected due to too many boolean clauses");
@@ -984,7 +985,7 @@
*/
public void testPrecedence() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new WhitespaceAnalyzer());
+ qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
Query query1 = qp.parse("A AND B OR C AND D", "field");
Query query2 = qp.parse("+A +B +C +D", "field");
@@ -995,7 +996,7 @@
public void testLocalDateFormat() throws IOException, QueryNodeException {
RAMDirectory ramDir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true,
+ IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
@@ -1120,7 +1121,7 @@
public void testMatchAllDocs() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new WhitespaceAnalyzer());
+ qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field"));
assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field"));
@@ -1132,7 +1133,7 @@
private void assertHits(int expected, String query, IndexSearcher is)
throws IOException, QueryNodeException {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new WhitespaceAnalyzer());
+ qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
qp.setLocale(Locale.ENGLISH);
Query q = qp.parse(query, "date");
@@ -1163,6 +1164,7 @@
final TermAttribute term = addAttribute(TermAttribute.class);
@Override
public boolean incrementToken() {
+ clearAttributes();
if (upto == 4) {
return false;
}
Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original)
+++ lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Fri Feb 26 13:09:54 2010
@@ -114,6 +114,7 @@
public boolean incrementToken() throws IOException {
if (inPhrase) {
inPhrase = false;
+ clearAttributes();
termAtt.setTermBuffer("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
@@ -138,7 +139,7 @@
/** Filters LowerCaseTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new LowerCaseTokenizer(reader));
+ return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader));
}
}
@@ -216,7 +217,7 @@
public QueryParserWrapper getParser(Analyzer a) throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
QueryParserWrapper qp = new QueryParserWrapper("field", a);
qp.setDefaultOperator(QueryParserWrapper.OR_OPERATOR);
return qp;
@@ -301,7 +302,7 @@
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new SimpleAnalyzer();
+ a = new SimpleAnalyzer(Version.LUCENE_CURRENT);
QueryParserWrapper qp = new QueryParserWrapper("field", a);
qp.setDefaultOperator(QueryParserWrapper.AND_OPERATOR);
return qp.parse(query);
@@ -328,9 +329,9 @@
public void testSimple() throws Exception {
assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
assertQueryEquals("term term term", null, "term term term");
- assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(),
+ assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(Version.LUCENE_CURRENT),
"t�rm term term");
- assertQueryEquals("�mlaut", new WhitespaceAnalyzer(), "�mlaut");
+ assertQueryEquals("�mlaut", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "�mlaut");
assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:");
@@ -395,7 +396,7 @@
}
public void testPunct() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -551,7 +552,7 @@
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod());
QueryParserWrapper qp = new QueryParserWrapper("field",
- new SimpleAnalyzer());
+ new SimpleAnalyzer(Version.LUCENE_CURRENT));
qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod());
@@ -570,7 +571,7 @@
public void testFarsiRangeCollating() throws Exception {
RAMDirectory ramDir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true,
+ IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES,
@@ -580,7 +581,7 @@
IndexSearcher is = new IndexSearcher(ramDir, true);
QueryParserWrapper qp = new QueryParserWrapper("content",
- new WhitespaceAnalyzer());
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
// RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
@@ -682,7 +683,7 @@
final String monthField = "month";
final String hourField = "hour";
QueryParserWrapper qp = new QueryParserWrapper("field",
- new SimpleAnalyzer());
+ new SimpleAnalyzer(Version.LUCENE_CURRENT));
// Don't set any date resolution and verify if DateField is used
assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
@@ -726,7 +727,7 @@
}
public void testEscaped() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
/*
* assertQueryEquals("\\[brackets", a, "\\[brackets");
@@ -823,7 +824,7 @@
}
public void testQueryStringEscaping() throws Exception {
- Analyzer a = new WhitespaceAnalyzer();
+ Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT);
assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
@@ -934,7 +935,7 @@
public void testCustomQueryParserWildcard() {
try {
- new QPTestParser("contents", new WhitespaceAnalyzer()).parse("a?t");
+ new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t");
fail("Wildcard queries should not be allowed");
} catch (ParseException expected) {
// expected exception
@@ -943,7 +944,7 @@
public void testCustomQueryParserFuzzy() throws Exception {
try {
- new QPTestParser("contents", new WhitespaceAnalyzer()).parse("xunit~");
+ new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~");
fail("Fuzzy queries should not be allowed");
} catch (ParseException expected) {
// expected exception
@@ -954,7 +955,7 @@
BooleanQuery.setMaxClauseCount(2);
try {
QueryParserWrapper qp = new QueryParserWrapper("field",
- new WhitespaceAnalyzer());
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
qp.parse("one two three");
fail("ParseException expected due to too many boolean clauses");
} catch (ParseException expected) {
@@ -967,7 +968,7 @@
*/
public void testPrecedence() throws Exception {
QueryParserWrapper qp = new QueryParserWrapper("field",
- new WhitespaceAnalyzer());
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
Query query1 = qp.parse("A AND B OR C AND D");
Query query2 = qp.parse("+A +B +C +D");
@@ -977,7 +978,7 @@
public void testLocalDateFormat() throws IOException, ParseException {
RAMDirectory ramDir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true,
+ IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
@@ -1094,7 +1095,7 @@
public void testMatchAllDocs() throws Exception {
QueryParserWrapper qp = new QueryParserWrapper("field",
- new WhitespaceAnalyzer());
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
assertEquals(new MatchAllDocsQuery(), qp.parse("*:*"));
assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)"));
BooleanQuery bq = (BooleanQuery) qp.parse("+*:* -*:*");
@@ -1105,7 +1106,7 @@
private void assertHits(int expected, String query, IndexSearcher is)
throws ParseException, IOException {
QueryParserWrapper qp = new QueryParserWrapper("date",
- new WhitespaceAnalyzer());
+ new WhitespaceAnalyzer(Version.LUCENE_CURRENT));
qp.setLocale(Locale.ENGLISH);
Query q = qp.parse(query);
ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
Modified: lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java?rev=916666&r1=916665&r2=916666&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java Fri Feb 26 13:09:54 2010
@@ -29,6 +29,7 @@
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.util.Version;
public class TestRegexQuery extends TestCase {
private IndexSearcher searcher;
@@ -39,7 +40,7 @@
public void setUp() {
RAMDirectory directory = new RAMDirectory();
try {
- IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true,
+ IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(Version.LUCENE_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.add(new Field(FN, "the quick brown fox jumps over the lazy dog", Field.Store.NO, Field.Index.ANALYZED));