You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2011/02/22 02:01:11 UTC
svn commit: r1073192 [18/32] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/...
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java Tue Feb 22 01:00:39 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search.spans;
import java.util.HashSet;
import java.util.Set;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -54,7 +55,7 @@ public class TestFieldMaskingSpanQuery e
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer= new RandomIndexWriter(random, directory);
+ RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
writer.addDocument(doc(new Field[] { field("id", "0")
,
@@ -111,7 +112,7 @@ public class TestFieldMaskingSpanQuery e
field("last", "jones") }));
reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Tue Feb 22 01:00:39 2011
@@ -57,7 +57,7 @@ public class TestNearSpansOrdered extend
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer= new RandomIndexWriter(random, directory);
+ RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
@@ -65,7 +65,7 @@ public class TestNearSpansOrdered extend
}
reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
protected String[] docFields = {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Tue Feb 22 01:00:39 2011
@@ -178,6 +178,7 @@ public class TestPayloadSpans extends Lu
spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), nestedSpanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 2, new int[]{3,3});
+ searcher.close();
closeIndexReader.close();
directory.close();
}
@@ -210,6 +211,7 @@ public class TestPayloadSpans extends Lu
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 1, new int[]{3});
+ searcher.close();
closeIndexReader.close();
directory.close();
}
@@ -247,6 +249,7 @@ public class TestPayloadSpans extends Lu
spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), nestedSpanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 2, new int[]{8, 8});
+ searcher.close();
closeIndexReader.close();
directory.close();
}
@@ -262,7 +265,7 @@ public class TestPayloadSpans extends Lu
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = new IndexSearcher(reader);
+ IndexSearcher is = newSearcher(reader);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -285,6 +288,7 @@ public class TestPayloadSpans extends Lu
assertEquals(2, payloadSet.size());
assertTrue(payloadSet.contains("a:Noise:10"));
assertTrue(payloadSet.contains("k:Noise:11"));
+ is.close();
reader.close();
directory.close();
}
@@ -299,7 +303,7 @@ public class TestPayloadSpans extends Lu
doc.add(new Field("content", new StringReader("a b a d k f a h i k a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = new IndexSearcher(reader);
+ IndexSearcher is = newSearcher(reader);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -321,6 +325,7 @@ public class TestPayloadSpans extends Lu
assertEquals(2, payloadSet.size());
assertTrue(payloadSet.contains("a:Noise:10"));
assertTrue(payloadSet.contains("k:Noise:11"));
+ is.close();
reader.close();
directory.close();
}
@@ -335,7 +340,7 @@ public class TestPayloadSpans extends Lu
doc.add(new Field("content", new StringReader("j k a l f k k p a t a k l k t a")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = new IndexSearcher(reader);
+ IndexSearcher is = newSearcher(reader);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -363,6 +368,7 @@ public class TestPayloadSpans extends Lu
}
assertTrue(payloadSet.contains("a:Noise:10"));
assertTrue(payloadSet.contains("k:Noise:11"));
+ is.close();
reader.close();
directory.close();
}
@@ -378,7 +384,7 @@ public class TestPayloadSpans extends Lu
IndexReader reader = writer.getReader();
writer.close();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
PayloadSpanUtil psu = new PayloadSpanUtil(searcher.getTopReaderContext());
@@ -389,6 +395,7 @@ public class TestPayloadSpans extends Lu
if(VERBOSE)
System.out.println(new String(bytes));
}
+ searcher.close();
reader.close();
directory.close();
}
@@ -443,7 +450,7 @@ public class TestPayloadSpans extends Lu
closeIndexReader = writer.getReader();
writer.close();
- IndexSearcher searcher = new IndexSearcher(closeIndexReader);
+ IndexSearcher searcher = newSearcher(closeIndexReader);
return searcher;
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java Tue Feb 22 01:00:39 2011
@@ -48,7 +48,7 @@ public class TestSpanFirstQuery extends
writer.addDocument(doc2);
IndexReader reader = writer.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
// user queries on "starts-with quick"
SpanQuery sfq = new SpanFirstQuery(new SpanTermQuery(new Term("field", "quick")), 1);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java Tue Feb 22 01:00:39 2011
@@ -53,7 +53,7 @@ public class TestSpanMultiTermQueryWrapp
iw.addDocument(doc);
reader = iw.getReader();
iw.close();
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Tue Feb 22 01:00:39 2011
@@ -53,7 +53,7 @@ public class TestSpans extends LuceneTes
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer= new RandomIndexWriter(random, directory);
+ RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(field, docFields[i], Field.Store.YES, Field.Index.ANALYZED));
@@ -61,7 +61,7 @@ public class TestSpans extends LuceneTes
}
reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
@@ -486,7 +486,7 @@ public class TestSpans extends LuceneTes
// Get searcher
final IndexReader reader = IndexReader.open(dir, true);
- final IndexSearcher searcher = new IndexSearcher(reader);
+ final IndexSearcher searcher = newSearcher(reader);
// Control (make sure docs indexed)
assertEquals(2, hitCount(searcher, "the"));
@@ -499,6 +499,7 @@ public class TestSpans extends LuceneTes
searcher.search(createSpan(0, true,
new SpanQuery[] {createSpan(4, false, "chased", "cat"),
createSpan("ate")}), 10).totalHits);
+ searcher.close();
reader.close();
dir.close();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java Tue Feb 22 01:00:39 2011
@@ -57,15 +57,16 @@ public class TestSpansAdvanced extends L
// create test index
mDirectory = newDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random,
- mDirectory, new MockAnalyzer(MockTokenizer.SIMPLE, true,
- MockTokenFilter.ENGLISH_STOPSET, true));
+ mDirectory, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(MockTokenizer.SIMPLE, true,
+ MockTokenFilter.ENGLISH_STOPSET, true)).setMergePolicy(newInOrderLogMergePolicy()));
addDocument(writer, "1", "I think it should work.");
addDocument(writer, "2", "I think it should work.");
addDocument(writer, "3", "I think it should work.");
addDocument(writer, "4", "I think it should work.");
reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java Tue Feb 22 01:00:39 2011
@@ -48,7 +48,7 @@ public class TestSpansAdvanced2 extends
final RandomIndexWriter writer = new RandomIndexWriter(random, mDirectory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(
MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true))
- .setOpenMode(OpenMode.APPEND));
+ .setOpenMode(OpenMode.APPEND).setMergePolicy(newInOrderLogMergePolicy()));
addDocument(writer, "A", "Should we, could we, would we?");
addDocument(writer, "B", "It should. Should it?");
addDocument(writer, "C", "It shouldn't.");
@@ -57,7 +57,7 @@ public class TestSpansAdvanced2 extends
writer.close();
// re-open the searcher since we added more docs
- searcher2 = new IndexSearcher(reader2);
+ searcher2 = newSearcher(reader2);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java Tue Feb 22 01:00:39 2011
@@ -271,7 +271,7 @@ public class TestBufferedIndexInput exte
assertEquals(reader.docFreq(bbb), 37);
dir.tweakBufferSizes();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
dir.tweakBufferSizes();
assertEquals(35, hits.length);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java Tue Feb 22 01:00:39 2011
@@ -39,8 +39,10 @@ public class TestFileSwitchDirectory ext
fileExtensions.add(IndexFileNames.FIELDS_EXTENSION);
fileExtensions.add(IndexFileNames.FIELDS_INDEX_EXTENSION);
- Directory primaryDir = new MockDirectoryWrapper(random, new RAMDirectory());
- Directory secondaryDir = new MockDirectoryWrapper(random, new RAMDirectory());
+ MockDirectoryWrapper primaryDir = new MockDirectoryWrapper(random, new RAMDirectory());
+ primaryDir.setCheckIndexOnClose(false); // only part of an index
+ MockDirectoryWrapper secondaryDir = new MockDirectoryWrapper(random, new RAMDirectory());
+ secondaryDir.setCheckIndexOnClose(false); // only part of an index
FileSwitchDirectory fsd = new FileSwitchDirectory(fileExtensions, primaryDir, secondaryDir, true);
IndexWriter writer = new IndexWriter(
@@ -49,7 +51,7 @@ public class TestFileSwitchDirectory ext
setMergePolicy(newLogMergePolicy(false))
);
TestIndexWriterReader.createIndexNoClose(true, "ram", writer);
- IndexReader reader = IndexReader.open(writer);
+ IndexReader reader = IndexReader.open(writer, true);
assertEquals(100, reader.maxDoc());
writer.commit();
// we should see only fdx,fdt files here
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java Tue Feb 22 01:00:39 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.store;
import java.io.File;
import java.util.Random;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -58,7 +59,7 @@ public class TestMultiMMap extends Lucen
// we will map a lot, try to turn on the unmap hack
if (MMapDirectory.UNMAP_SUPPORTED)
dir.setUseUnmap(true);
- RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
Document doc = new Document();
Field docid = newField("docid", "0", Field.Store.YES, Field.Index.NOT_ANALYZED);
Field junk = newField("junk", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestRAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestRAMDirectory.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestRAMDirectory.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/store/TestRAMDirectory.java Tue Feb 22 01:00:39 2011
@@ -82,7 +82,7 @@ public class TestRAMDirectory extends Lu
assertEquals(docsToAdd, reader.numDocs());
// open search zo check if all doc's are there
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
// search for all documents
for (int i = 0; i < docsToAdd; i++) {
@@ -139,18 +139,6 @@ public class TestRAMDirectory extends Lu
writer.close();
}
-
- public void testSerializable() throws IOException {
- Directory dir = new RAMDirectory();
- ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
- assertEquals("initially empty", 0, bos.size());
- ObjectOutput out = new ObjectOutputStream(bos);
- int headerSize = bos.size();
- out.writeObject(dir);
- out.close();
- assertTrue("contains more then just header", headerSize < bos.size());
- }
-
@Override
public void tearDown() throws Exception {
// cleanup
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestBytesRefHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestBytesRefHash.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestBytesRefHash.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestBytesRefHash.java Tue Feb 22 01:00:39 2011
@@ -40,6 +40,7 @@ public class TestBytesRefHash extends Lu
/**
*/
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestDoubleBarrelLRUCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestDoubleBarrelLRUCache.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestDoubleBarrelLRUCache.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestDoubleBarrelLRUCache.java Tue Feb 22 01:00:39 2011
@@ -145,14 +145,17 @@ public class TestDoubleBarrelLRUCache ex
this.value = value;
}
+ @Override
public boolean equals(Object other) {
return this.value.equals(((CloneableObject) other).value);
}
+ @Override
public int hashCode() {
return value.hashCode();
}
+ @Override
public Object clone() {
return new CloneableObject(value);
}
@@ -165,14 +168,17 @@ public class TestDoubleBarrelLRUCache ex
this.value = value;
}
+ @Override
public boolean equals(Object other) {
return this.value.equals(((CloneableInteger) other).value);
}
+ @Override
public int hashCode() {
return value.hashCode();
}
+ @Override
public Object clone() {
return new CloneableInteger(value);
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java Tue Feb 22 01:00:39 2011
@@ -31,6 +31,7 @@ public class TestRecyclingByteBlockAlloc
/**
*/
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java Tue Feb 22 01:00:39 2011
@@ -20,9 +20,11 @@ package org.apache.lucene.util.automaton
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.PrintStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -59,11 +61,13 @@ public class TestFSTs extends LuceneTest
private MockDirectoryWrapper dir;
+ @Override
public void setUp() throws IOException {
dir = newDirectory();
dir.setPreventDoubleWrite(false);
}
+ @Override
public void tearDown() throws IOException {
dir.close();
}
@@ -443,9 +447,9 @@ public class TestFSTs extends LuceneTest
}
if (VERBOSE && pairs.size() <= 20 && fst != null) {
- PrintStream ps = new PrintStream("out.dot");
- Util.toDot(fst, ps);
- ps.close();
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Util.toDot(fst, w, false, false);
+ w.close();
System.out.println("SAVED out.dot");
}
@@ -958,7 +962,7 @@ public class TestFSTs extends LuceneTest
writer.addDocument(doc);
docCount++;
}
- IndexReader r = IndexReader.open(writer);
+ IndexReader r = IndexReader.open(writer, true);
writer.close();
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(random.nextBoolean());
Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, outputs);
@@ -1126,9 +1130,9 @@ public class TestFSTs extends LuceneTest
System.out.println(ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount() + " arcs; " + fst.getArcWithOutputCount() + " arcs w/ output; tot size " + fst.sizeInBytes());
if (fst.getNodeCount() < 100) {
- PrintStream ps = new PrintStream("out.dot");
- Util.toDot(fst, ps);
- ps.close();
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Util.toDot(fst, w, false, false);
+ w.close();
System.out.println("Wrote FST to out.dot");
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt Tue Feb 22 01:00:39 2011
@@ -80,6 +80,9 @@ New Features
- o.a.l.analysis.StopwordAnalyzerBase -> o.a.l.analysis.util.StopwordAnalyzerBase
- o.a.l.analysis.WordListLoader -> o.a.l.analysis.util.WordListLoader
+ * SOLR-1057: Add PathHierarchyTokenizer that represents file path hierarchies as synonyms of
+ /something, /something/something, /something/something/else. (Ryan McKinley, Koji Sekiguchi)
+
Build
* LUCENE-2413: All analyzers in contrib/analyzers and contrib/icu were moved to the
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/build.xml?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/build.xml (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/build.xml Tue Feb 22 01:00:39 2011
@@ -30,6 +30,7 @@
<path id="test.classpath">
<path refid="classpath"/>
+ <pathelement location="../../../lucene/build/classes/test-framework"/>
<pathelement location="../../../lucene/build/classes/test/"/>
<path refid="junit-path"/>
<pathelement location="${build.dir}/classes/java"/>
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java Tue Feb 22 01:00:39 2011
@@ -82,7 +82,7 @@ public final class BrazilianAnalyzer ext
private Set<?> excltable = Collections.emptySet();
/**
- * Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
+ * Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}).
*/
public BrazilianAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java Tue Feb 22 01:00:39 2011
@@ -101,6 +101,7 @@ public class HTMLStripCharFilter extends
if (len>0) {
return pushed.charAt(len-1);
}
+ numRead++;
int ch = input.read();
push(ch);
return ch;
@@ -672,6 +673,7 @@ public class HTMLStripCharFilter extends
+ @Override
public int read() throws IOException {
// TODO: Do we ever want to preserve CDATA sections?
// where do we have to worry about them?
@@ -740,6 +742,7 @@ public class HTMLStripCharFilter extends
}
+ @Override
public int read(char cbuf[], int off, int len) throws IOException {
int i=0;
for (i=0; i<len; i++) {
@@ -754,6 +757,7 @@ public class HTMLStripCharFilter extends
return i;
}
+ @Override
public void close() throws IOException {
input.close();
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java Tue Feb 22 01:00:39 2011
@@ -65,7 +65,7 @@ public final class CJKAnalyzer extends S
}
/**
- * Builds an analyzer which removes words in {@link #STOP_WORDS}.
+ * Builds an analyzer which removes words in {@link #getDefaultStopSet()}.
*/
public CJKAnalyzer(Version matchVersion) {
this(matchVersion, DefaultSetHolder.DEFAULT_STOP_SET);
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java Tue Feb 22 01:00:39 2011
@@ -116,6 +116,7 @@ public final class CommonGramsFilter ext
* eliminate the middle bigram "of-the"and save a disk seek and a whole set of
* position lookups.
*/
+ @Override
public boolean incrementToken() throws IOException {
// get the next piece of input
if (savedState != null) {
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java Tue Feb 22 01:00:39 2011
@@ -62,6 +62,7 @@ public final class CommonGramsQueryFilte
/**
* {@inheritDoc}
*/
+ @Override
public void reset() throws IOException {
super.reset();
previous = null;
@@ -76,6 +77,7 @@ public final class CommonGramsQueryFilte
* <li>output:"the-rain", "rain-in" ,"in-spain", "falls", "mainly"
* </ul>
*/
+ @Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
State current = captureState();
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java Tue Feb 22 01:00:39 2011
@@ -18,14 +18,12 @@
package org.apache.lucene.analysis.compound.hyphenation;
-import java.io.Serializable;
-
/**
* This class implements a simple byte vector with access to the underlying
* array.
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class ByteVector implements Serializable {
+public class ByteVector {
/**
* Capacity increment size
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java Tue Feb 22 01:00:39 2011
@@ -17,15 +17,13 @@
package org.apache.lucene.analysis.compound.hyphenation;
-import java.io.Serializable;
-
/**
* This class implements a simple char vector with access to the underlying
* array.
*
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class CharVector implements Cloneable, Serializable {
+public class CharVector implements Cloneable {
/**
* Capacity increment size
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java Tue Feb 22 01:00:39 2011
@@ -17,8 +17,6 @@
package org.apache.lucene.analysis.compound.hyphenation;
-import java.io.Serializable;
-
/**
* This class represents a hyphen. A 'full' hyphen is made of 3 parts: the
* pre-break text, post-break text and no-break. If no line-break is generated
@@ -32,7 +30,7 @@ import java.io.Serializable;
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class Hyphen implements Serializable {
+public class Hyphen {
public String preBreak;
public String noBreak;
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java Tue Feb 22 01:00:39 2011
@@ -18,7 +18,6 @@
package org.apache.lucene.analysis.compound.hyphenation;
import java.io.File;
-import java.io.Serializable;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -31,10 +30,7 @@ import org.xml.sax.InputSource;
*
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class HyphenationTree extends TernaryTree implements PatternConsumer,
- Serializable {
-
- private static final long serialVersionUID = -7842107987915665573L;
+public class HyphenationTree extends TernaryTree implements PatternConsumer {
/**
* value space: stores the interletter values
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java Tue Feb 22 01:00:39 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.compo
import java.util.Enumeration;
import java.util.Stack;
-import java.io.Serializable;
/**
* <h2>Ternary Search Tree.</h2>
@@ -63,7 +62,7 @@ import java.io.Serializable;
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class TernaryTree implements Cloneable, Serializable {
+public class TernaryTree implements Cloneable {
/**
* We use 4 arrays to represent a node. I guess I should have created a proper
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java Tue Feb 22 01:00:39 2011
@@ -22,10 +22,9 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
-import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.util.Version;
@@ -42,14 +41,10 @@ import org.apache.lucene.util.Version;
* increments are preserved
* </ul>
*/
-public final class StopFilter extends TokenFilter {
+public final class StopFilter extends FilteringTokenFilter {
private final CharArraySet stopWords;
- private boolean enablePositionIncrements = true;
-
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-
/**
* Construct a token stream filtering the given input. If
@@ -75,7 +70,7 @@ public final class StopFilter extends To
*/
public StopFilter(Version matchVersion, TokenStream input, Set<?> stopWords, boolean ignoreCase)
{
- super(input);
+ super(true, input);
this.stopWords = stopWords instanceof CharArraySet ? (CharArraySet) stopWords : new CharArraySet(matchVersion, stopWords, ignoreCase);
}
@@ -157,48 +152,8 @@ public final class StopFilter extends To
* Returns the next input Token whose term() is not a stop word.
*/
@Override
- public final boolean incrementToken() throws IOException {
- // return the first non-stop word found
- int skippedPositions = 0;
- while (input.incrementToken()) {
- if (!stopWords.contains(termAtt.buffer(), 0, termAtt.length())) {
- if (enablePositionIncrements) {
- posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
- }
- return true;
- }
- skippedPositions += posIncrAtt.getPositionIncrement();
- }
- // reached EOS -- return false
- return false;
- }
-
- /**
- * @see #setEnablePositionIncrements(boolean)
- */
- public boolean getEnablePositionIncrements() {
- return enablePositionIncrements;
+ protected boolean accept() throws IOException {
+ return !stopWords.contains(termAtt.buffer(), 0, termAtt.length());
}
- /**
- * If <code>true</code>, this StopFilter will preserve
- * positions of the incoming tokens (ie, accumulate and
- * set position increments of the removed stop tokens).
- * Generally, <code>true</code> is best as it does not
- * lose information (positions of the original tokens)
- * during indexing.
- *
- * Default is true.
- *
- * <p> When set, when a token is stopped
- * (omitted), the position increment of the following
- * token is incremented.
- *
- * <p> <b>NOTE</b>: be sure to also
- * set {@link QueryParser#setEnablePositionIncrements} if
- * you use QueryParser to create queries.
- */
- public void setEnablePositionIncrements(boolean enable) {
- this.enablePositionIncrements = enable;
- }
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java Tue Feb 22 01:00:39 2011
@@ -86,7 +86,7 @@ public final class CzechAnalyzer extends
private final Set<?> stemExclusionTable;
/**
- * Builds an analyzer with the default stop words ({@link #CZECH_STOP_WORDS}).
+ * Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}).
*
* @param matchVersion Lucene version to match See
* {@link <a href="#version">above</a>}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java Tue Feb 22 01:00:39 2011
@@ -32,6 +32,7 @@ public class PersianCharFilter extends C
super(in);
}
+ @Override
public int read(char[] cbuf, int off, int len) throws IOException {
final int charsRead = super.read(cbuf, off, len);
if (charsRead > 0) {
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java Tue Feb 22 01:00:39 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -30,22 +31,19 @@ import org.apache.lucene.analysis.util.C
*
* @since solr 1.3
*/
-public final class KeepWordFilter extends TokenFilter {
+public final class KeepWordFilter extends FilteringTokenFilter {
private final CharArraySet words;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/** The words set passed to this constructor will be directly used by this filter
* and should not be modified, */
- public KeepWordFilter(TokenStream in, CharArraySet words) {
- super(in);
+ public KeepWordFilter(boolean enablePositionIncrements, TokenStream in, CharArraySet words) {
+ super(enablePositionIncrements, in);
this.words = words;
}
@Override
- public boolean incrementToken() throws IOException {
- while (input.incrementToken()) {
- if (words.contains(termAtt.buffer(), 0, termAtt.length())) return true;
- }
- return false;
+ public boolean accept() throws IOException {
+ return words.contains(termAtt.buffer(), 0, termAtt.length());
}
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java Tue Feb 22 01:00:39 2011
@@ -74,10 +74,12 @@ public final class KeywordMarkerFilter e
@Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- keywordAttr.setKeyword(keywordSet.contains(termAtt.buffer(), 0,
- termAtt.length()));
+ if (keywordSet.contains(termAtt.buffer(), 0, termAtt.length())) {
+ keywordAttr.setKeyword(true);
+ }
return true;
- } else
+ } else {
return false;
+ }
}
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java Tue Feb 22 01:00:39 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
@@ -29,7 +30,7 @@ import org.apache.lucene.analysis.tokena
* Note: Length is calculated as the number of UTF-16 code units.
* </p>
*/
-public final class LengthFilter extends TokenFilter {
+public final class LengthFilter extends FilteringTokenFilter {
private final int min;
private final int max;
@@ -40,27 +41,15 @@ public final class LengthFilter extends
* Build a filter that removes words that are too long or too
* short from the text.
*/
- public LengthFilter(TokenStream in, int min, int max)
- {
- super(in);
+ public LengthFilter(boolean enablePositionIncrements, TokenStream in, int min, int max) {
+ super(enablePositionIncrements, in);
this.min = min;
this.max = max;
}
- /**
- * Returns the next input Token whose term() is the right len
- */
@Override
- public final boolean incrementToken() throws IOException {
- // return the first non-stop word found
- while (input.incrementToken()) {
- int len = termAtt.length();
- if (len >= min && len <= max) {
- return true;
- }
- // note: else we ignore it but should we index each part of it?
- }
- // reached EOS -- return false
- return false;
+ public boolean accept() throws IOException {
+ final int len = termAtt.length();
+ return (len >= min && len <= max);
}
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Tue Feb 22 01:00:39 2011
@@ -109,7 +109,7 @@ public final class DutchAnalyzer extends
private final Version matchVersion;
/**
- * Builds an analyzer with the default stop words ({@link #DUTCH_STOP_WORDS})
+ * Builds an analyzer with the default stop words ({@link #getDefaultStopSet()})
* and a few default entries for the stem exclusion table.
*
*/
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java?rev=1073192&r1=1073113&r2=1073192&view=diff
==============================================================================
(empty)
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java Tue Feb 22 01:00:39 2011
@@ -113,6 +113,7 @@ public class PatternReplaceCharFilter ex
}
}
+ @Override
public int read() throws IOException {
while( prepareReplaceBlock() ){
return replaceBlockBuffer.charAt( replaceBlockBufferOffset++ );
@@ -120,6 +121,7 @@ public class PatternReplaceCharFilter ex
return -1;
}
+ @Override
public int read(char[] cbuf, int off, int len) throws IOException {
char[] tmp = new char[len];
int l = input.read(tmp, 0, len);
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro Tue Feb 22 01:00:39 2011
@@ -15,8 +15,8 @@
*/
// Generated from IANA Root Zone Database <http://www.internic.net/zones/root.zone>
-// file version from Wednesday, January 5, 2011 12:34:09 PM UTC
-// generated on Thursday, January 6, 2011 5:09:41 AM UTC
+// file version from Wednesday, February 9, 2011 12:34:10 PM UTC
+// generated on Wednesday, February 9, 2011 4:45:18 PM UTC
// by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros
ASCIITLD = "." (
@@ -285,13 +285,19 @@ ASCIITLD = "." (
| [wW][sS]
| [xX][nN]--0[zZ][wW][mM]56[dD]
| [xX][nN]--11[bB]5[bB][sS]3[aA]9[aA][jJ]6[gG]
+ | [xX][nN]--3[eE]0[bB]707[eE]
+ | [xX][nN]--45[bB][rR][jJ]9[cC]
| [xX][nN]--80[aA][kK][hH][bB][yY][kK][nN][jJ]4[fF]
| [xX][nN]--9[tT]4[bB]11[yY][iI]5[aA]
+ | [xX][nN]--[cC][lL][cC][hH][cC]0[eE][aA]0[bB]2[gG]2[aA]9[gG][cC][dD]
| [xX][nN]--[dD][eE][bB][aA]0[aA][dD]
| [xX][nN]--[fF][iI][qQ][sS]8[sS]
| [xX][nN]--[fF][iI][qQ][zZ]9[sS]
+ | [xX][nN]--[fF][pP][cC][rR][jJ]9[cC]3[dD]
| [xX][nN]--[fF][zZ][cC]2[cC]9[eE]2[cC]
| [xX][nN]--[gG]6[wW]251[dD]
+ | [xX][nN]--[gG][eE][cC][rR][jJ]9[cC]
+ | [xX][nN]--[hH]2[bB][rR][jJ]9[cC]
| [xX][nN]--[hH][gG][bB][kK]6[aA][jJ]7[fF]53[bB][bB][aA]
| [xX][nN]--[hH][lL][cC][jJ]6[aA][yY][aA]9[eE][sS][cC]7[aA]
| [xX][nN]--[jJ]6[wW]193[gG]
@@ -301,13 +307,18 @@ ASCIITLD = "." (
| [xX][nN]--[kK][pP][rR][yY]57[dD]
| [xX][nN]--[mM][gG][bB][aA][aA][mM]7[aA]8[hH]
| [xX][nN]--[mM][gG][bB][aA][yY][hH]7[gG][pP][aA]
+ | [xX][nN]--[mM][gG][bB][bB][hH]1[aA]71[eE]
| [xX][nN]--[mM][gG][bB][eE][rR][pP]4[aA]5[dD]4[aA][rR]
| [xX][nN]--[oO]3[cC][wW]4[hH]
+ | [xX][nN]--[oO][gG][bB][pP][fF]8[fF][lL]
| [xX][nN]--[pP]1[aA][iI]
| [xX][nN]--[pP][gG][bB][sS]0[dD][hH]
+ | [xX][nN]--[sS]9[bB][rR][jJ]9[cC]
| [xX][nN]--[wW][gG][bB][hH]1[cC]
| [xX][nN]--[wW][gG][bB][lL]6[aA]
| [xX][nN]--[xX][kK][cC]2[aA][lL]3[hH][yY][eE]2[aA]
+ | [xX][nN]--[xX][kK][cC]2[dD][lL]3[aA]5[eE][eE]0[hH]
+ | [xX][nN]--[yY][fF][rR][oO]4[iI]67[oO]
| [xX][nN]--[yY][gG][bB][iI]2[aA][mM][mM][xX]
| [xX][nN]--[zZ][cC][kK][zZ][aA][hH]
| [yY][eE]
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Tue Feb 22 01:00:39 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 12/4/10 7:24 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 2/9/11 11:45 AM */
package org.apache.lucene.analysis.standard;
@@ -26,14 +26,15 @@ WARNING: if you change ClassicTokenizerI
*/
+import java.io.Reader;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 12/4/10 7:24 PM from the specification file
- * <tt>C:/cygwin/home/us/svn/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 2/9/11 11:45 AM from the specification file
+ * <tt>C:/Users/rmuir/workspace/lucene-2911/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
*/
class ClassicTokenizerImpl implements StandardTokenizerInterface {
@@ -681,44 +682,44 @@ public final void getText(CharTermAttrib
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
- case 5:
- { return NUM;
+ case 10:
+ { return EMAIL;
}
case 11: break;
- case 9:
- { return ACRONYM;
+ case 2:
+ { return ALPHANUM;
}
case 12: break;
- case 7:
- { return COMPANY;
+ case 4:
+ { return HOST;
}
case 13: break;
- case 10:
- { return EMAIL;
- }
- case 14: break;
case 1:
{ /* ignore */
}
+ case 14: break;
+ case 8:
+ { return ACRONYM_DEP;
+ }
case 15: break;
- case 6:
- { return APOSTROPHE;
+ case 5:
+ { return NUM;
}
case 16: break;
- case 3:
- { return CJ;
+ case 9:
+ { return ACRONYM;
}
case 17: break;
- case 8:
- { return ACRONYM_DEP;
+ case 7:
+ { return COMPANY;
}
case 18: break;
- case 2:
- { return ALPHANUM;
+ case 6:
+ { return APOSTROPHE;
}
case 19: break;
- case 4:
- { return HOST;
+ case 3:
+ { return CJ;
}
case 20: break;
default:
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Tue Feb 22 01:00:39 2011
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-// Generated using ICU4J 4.6.0.0 on Thursday, January 6, 2011 7:02:52 PM UTC
+// Generated using ICU4J 4.6.0.0 on Wednesday, February 9, 2011 4:45:11 PM UTC
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1073192&r1=1073191&r2=1073192&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Tue Feb 22 01:00:39 2011
@@ -78,6 +78,8 @@ public final class StandardTokenizer ext
public static final int SOUTHEAST_ASIAN = 9;
public static final int IDEOGRAPHIC = 10;
public static final int HIRAGANA = 11;
+ public static final int KATAKANA = 12;
+ public static final int HANGUL = 13;
/** String token types that correspond to token type int constants */
public static final String [] TOKEN_TYPES = new String [] {
@@ -92,7 +94,9 @@ public final class StandardTokenizer ext
"<ACRONYM_DEP>",
"<SOUTHEAST_ASIAN>",
"<IDEOGRAPHIC>",
- "<HIRAGANA>"
+ "<HIRAGANA>",
+ "<KATAKANA>",
+ "<HANGUL>"
};
private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;