You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2010/05/04 13:18:50 UTC
svn commit: r940806 [4/5] - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/search/
lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/analys...
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Tue May 4 11:18:46 2010
@@ -2,7 +2,7 @@ package org.apache.lucene.analysis;
import java.io.StringReader;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -30,19 +30,19 @@ public class TestPerFieldAnalzyerWrapper
TokenStream tokenStream = analyzer.tokenStream("field",
new StringReader(text));
- TermAttribute termAtt = tokenStream.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
assertTrue(tokenStream.incrementToken());
assertEquals("WhitespaceAnalyzer does not lowercase",
"Qwerty",
- termAtt.term());
+ termAtt.toString());
tokenStream = analyzer.tokenStream("special",
new StringReader(text));
- termAtt = tokenStream.getAttribute(TermAttribute.class);
+ termAtt = tokenStream.getAttribute(CharTermAttribute.class);
assertTrue(tokenStream.incrementToken());
assertEquals("SimpleAnalyzer lowercases",
"qwerty",
- termAtt.term());
+ termAtt.toString());
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Tue May 4 11:18:46 2010
@@ -224,4 +224,12 @@ public class TestStandardAnalyzer extend
"<ALPHANUM>", "<NUM>", "<HOST>", "<NUM>", "<ALPHANUM>",
"<ALPHANUM>", "<HOST>"});
}
+
+ public void testJava14BWCompatibility() throws Exception {
+ StandardAnalyzer sa = new StandardAnalyzer(Version.LUCENE_30);
+ assertAnalyzesTo(sa, "test\u02C6test", new String[] { "test", "test" });
+ sa = new StandardAnalyzer(Version.LUCENE_31);
+ assertAnalyzesTo(sa, "test\u02C6test", new String[] { "test\u02C6test" });
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Tue May 4 11:18:46 2010
@@ -18,7 +18,7 @@ package org.apache.lucene.analysis;
*/
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import java.io.StringReader;
@@ -51,10 +51,10 @@ public class TestStopAnalyzer extends Ba
StringReader reader = new StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.tokenStream("test", reader);
assertTrue(stream != null);
- TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
while (stream.incrementToken()) {
- assertFalse(inValidTokens.contains(termAtt.term()));
+ assertFalse(inValidTokens.contains(termAtt.toString()));
}
}
@@ -67,11 +67,11 @@ public class TestStopAnalyzer extends Ba
StringReader reader = new StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
- TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
- String text = termAtt.term();
+ String text = termAtt.toString();
assertFalse(stopWordsSet.contains(text));
assertEquals(1,posIncrAtt.getPositionIncrement()); // in 2.4 stop tokenizer does not apply increments.
}
@@ -88,11 +88,11 @@ public class TestStopAnalyzer extends Ba
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
int i = 0;
- TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
- String text = termAtt.term();
+ String text = termAtt.toString();
assertFalse(stopWordsSet.contains(text));
assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java Tue May 4 11:18:46 2010
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis;
*/
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.English;
import org.apache.lucene.util.Version;
@@ -37,11 +37,11 @@ public class TestStopFilter extends Base
StringReader reader = new StringReader("Now is The Time");
Set<String> stopWords = new HashSet<String>(Arrays.asList("is", "the", "Time"));
TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopWords, false);
- final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
+ assertEquals("Now", termAtt.toString());
assertTrue(stream.incrementToken());
- assertEquals("The", termAtt.term());
+ assertEquals("The", termAtt.toString());
assertFalse(stream.incrementToken());
}
@@ -49,9 +49,9 @@ public class TestStopFilter extends Base
StringReader reader = new StringReader("Now is The Time");
Set<Object> stopWords = new HashSet<Object>(Arrays.asList( "is", "the", "Time" ));
TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopWords, true);
- final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
+ assertEquals("Now", termAtt.toString());
assertFalse(stream.incrementToken());
}
@@ -60,11 +60,11 @@ public class TestStopFilter extends Base
String[] stopWords = new String[] { "is", "the", "Time" };
Set<Object> stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
- final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+ final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
+ assertEquals("Now", termAtt.toString());
assertTrue(stream.incrementToken());
- assertEquals("The", termAtt.term());
+ assertEquals("The", termAtt.toString());
assertFalse(stream.incrementToken());
}
@@ -117,13 +117,13 @@ public class TestStopFilter extends Base
private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
stpf.setEnablePositionIncrements(enableIcrements);
- TermAttribute termAtt = stpf.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = stpf.getAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stpf.getAttribute(PositionIncrementAttribute.class);
for (int i=0; i<20; i+=3) {
assertTrue(stpf.incrementToken());
log("Token "+i+": "+stpf);
String w = English.intToEnglish(i).trim();
- assertEquals("expecting token "+i+" to be "+w,w,termAtt.term());
+ assertEquals("expecting token "+i+" to be "+w,w,termAtt.toString());
assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,posIncrAtt.getPositionIncrement());
}
assertFalse(stpf.incrementToken());
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java Tue May 4 11:18:46 2010
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.English;
import java.io.IOException;
@@ -59,16 +59,16 @@ public class TestTeeSinkTokenFilter exte
static final TeeSinkTokenFilter.SinkFilter theFilter = new TeeSinkTokenFilter.SinkFilter() {
@Override
public boolean accept(AttributeSource a) {
- TermAttribute termAtt = a.getAttribute(TermAttribute.class);
- return termAtt.term().equalsIgnoreCase("The");
+ CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+ return termAtt.toString().equalsIgnoreCase("The");
}
};
static final TeeSinkTokenFilter.SinkFilter dogFilter = new TeeSinkTokenFilter.SinkFilter() {
@Override
public boolean accept(AttributeSource a) {
- TermAttribute termAtt = a.getAttribute(TermAttribute.class);
- return termAtt.term().equalsIgnoreCase("Dogs");
+ CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+ return termAtt.toString().equalsIgnoreCase("Dogs");
}
};
@@ -135,8 +135,8 @@ public class TestTeeSinkTokenFilter exte
TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(100));
teeStream.consumeAllTokens();
TokenStream stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), 100);
- TermAttribute tfTok = stream.addAttribute(TermAttribute.class);
- TermAttribute sinkTok = sink.addAttribute(TermAttribute.class);
+ CharTermAttribute tfTok = stream.addAttribute(CharTermAttribute.class);
+ CharTermAttribute sinkTok = sink.addAttribute(CharTermAttribute.class);
for (int i=0; stream.incrementToken(); i++) {
assertTrue(sink.incrementToken());
assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.equals(sinkTok) == true);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java Tue May 4 11:18:46 2010
@@ -36,6 +36,8 @@ public class TestToken extends LuceneTes
char[] content = "hello".toCharArray();
t.setTermBuffer(content, 0, content.length);
assertNotSame(t.termBuffer(), content);
+ assertEquals(0, t.startOffset());
+ assertEquals(0, t.endOffset());
assertEquals("hello", t.term());
assertEquals("word", t.type());
assertEquals(0, t.getFlags());
@@ -43,20 +45,28 @@ public class TestToken extends LuceneTes
t = new Token(6, 22);
t.setTermBuffer(content, 0, content.length);
assertEquals("hello", t.term());
- assertEquals("(hello,6,22)", t.toString());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
assertEquals("word", t.type());
assertEquals(0, t.getFlags());
t = new Token(6, 22, 7);
t.setTermBuffer(content, 0, content.length);
assertEquals("hello", t.term());
- assertEquals("(hello,6,22)", t.toString());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
+ assertEquals("word", t.type());
assertEquals(7, t.getFlags());
t = new Token(6, 22, "junk");
t.setTermBuffer(content, 0, content.length);
assertEquals("hello", t.term());
- assertEquals("(hello,6,22,type=junk)", t.toString());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
+ assertEquals("junk", t.type());
assertEquals(0, t.getFlags());
}
@@ -142,10 +152,10 @@ public class TestToken extends LuceneTes
char[] b = {'a', 'l', 'o', 'h', 'a'};
Token t = new Token("", 0, 5);
t.setTermBuffer(b, 0, 5);
- assertEquals("(aloha,0,5)", t.toString());
+ assertEquals("aloha", t.toString());
t.setTermBuffer("hi there");
- assertEquals("(hi there,0,5)", t.toString());
+ assertEquals("hi there", t.toString());
}
public void testTermBufferEquals() throws Exception {
@@ -234,8 +244,8 @@ public class TestToken extends LuceneTes
assertTrue("TypeAttribute is not implemented by SenselessAttributeImpl",
ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
- assertTrue("TermAttribute is not implemented by Token",
- ts.addAttribute(TermAttribute.class) instanceof Token);
+ assertTrue("CharTermAttribute is not implemented by Token",
+ ts.addAttribute(CharTermAttribute.class) instanceof Token);
assertTrue("OffsetAttribute is not implemented by Token",
ts.addAttribute(OffsetAttribute.class) instanceof Token);
assertTrue("FlagsAttribute is not implemented by Token",
Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java (from r924791, lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java&p1=lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java&r1=924791&r2=940806&rev=940806&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java Tue May 4 11:18:46 2010
@@ -22,6 +22,7 @@ import java.nio.CharBuffer;
import java.util.Formatter;
import java.util.Locale;
import java.util.regex.Pattern;
+import java.util.Random;
public class TestCharTermAttributeImpl extends LuceneTestCase {
@@ -157,21 +158,155 @@ public class TestCharTermAttributeImpl e
assertEquals("12345678", t.toString());
t.append('9');
assertEquals("123456789", t.toString());
- t.append("0");
+ t.append((CharSequence) "0");
assertEquals("1234567890", t.toString());
- t.append("0123456789", 1, 3);
+ t.append((CharSequence) "0123456789", 1, 3);
assertEquals("123456789012", t.toString());
- t.append(CharBuffer.wrap("0123456789".toCharArray()), 3, 5);
+ t.append((CharSequence) CharBuffer.wrap("0123456789".toCharArray()), 3, 5);
assertEquals("12345678901234", t.toString());
- t.append(t);
+ t.append((CharSequence) t);
assertEquals("1234567890123412345678901234", t.toString());
- t.append(new StringBuilder("0123456789"), 5, 7);
+ t.append((CharSequence) new StringBuilder("0123456789"), 5, 7);
assertEquals("123456789012341234567890123456", t.toString());
- t.append(new StringBuffer(t));
+ t.append((CharSequence) new StringBuffer(t));
assertEquals("123456789012341234567890123456123456789012341234567890123456", t.toString());
// very wierd, to test if a subSlice is wrapped correct :)
- t.setEmpty().append(CharBuffer.wrap("0123456789".toCharArray(), 3, 5) /* "34" */, 1, 2);
+ CharBuffer buf = CharBuffer.wrap("0123456789".toCharArray(), 3, 5);
+ assertEquals("34567", buf.toString());
+ t.setEmpty().append((CharSequence) buf, 1, 2);
assertEquals("4", t.toString());
+ CharTermAttribute t2 = new CharTermAttributeImpl();
+ t2.append("test");
+ t.append((CharSequence) t2);
+ assertEquals("4test", t.toString());
+ t.append((CharSequence) t2, 1, 2);
+ assertEquals("4teste", t.toString());
+
+ try {
+ t.append((CharSequence) t2, 1, 5);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+
+ try {
+ t.append((CharSequence) t2, 1, 0);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+
+ t.append((CharSequence) null);
+ assertEquals("4testenull", t.toString());
+ }
+
+ public void testAppendableInterfaceWithLongSequences() {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ t.append((CharSequence) "01234567890123456789012345678901234567890123456789");
+ t.append((CharSequence) CharBuffer.wrap("01234567890123456789012345678901234567890123456789".toCharArray()), 3, 50);
+ assertEquals("0123456789012345678901234567890123456789012345678934567890123456789012345678901234567890123456789", t.toString());
+ t.setEmpty().append((CharSequence) new StringBuilder("01234567890123456789"), 5, 17);
+ assertEquals((CharSequence) "567890123456", t.toString());
+ t.append(new StringBuffer(t));
+ assertEquals((CharSequence) "567890123456567890123456", t.toString());
+ // very wierd, to test if a subSlice is wrapped correct :)
+ CharBuffer buf = CharBuffer.wrap("012345678901234567890123456789".toCharArray(), 3, 15);
+ assertEquals("345678901234567", buf.toString());
+ t.setEmpty().append(buf, 1, 14);
+ assertEquals("4567890123456", t.toString());
+
+ // finally use a completely custom CharSequence that is not catched by instanceof checks
+ final String longTestString = "012345678901234567890123456789";
+ t.append(new CharSequence() {
+ public char charAt(int i) { return longTestString.charAt(i); }
+ public int length() { return longTestString.length(); }
+ public CharSequence subSequence(int start, int end) { return longTestString.subSequence(start, end); }
+ public String toString() { return longTestString; }
+ });
+ assertEquals("4567890123456"+longTestString, t.toString());
+ }
+
+ public void testNonCharSequenceAppend() {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ t.append("0123456789");
+ t.append("0123456789");
+ assertEquals("01234567890123456789", t.toString());
+ t.append(new StringBuilder("0123456789"));
+ assertEquals("012345678901234567890123456789", t.toString());
+ CharTermAttribute t2 = new CharTermAttributeImpl();
+ t2.append("test");
+ t.append(t2);
+ assertEquals("012345678901234567890123456789test", t.toString());
+ t.append((String) null);
+ t.append((StringBuilder) null);
+ t.append((CharTermAttribute) null);
+ assertEquals("012345678901234567890123456789testnullnullnull", t.toString());
+ }
+
+ public void testExceptions() {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ t.append("test");
+ assertEquals("test", t.toString());
+
+ try {
+ t.charAt(-1);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+
+ try {
+ t.charAt(4);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+
+ try {
+ t.subSequence(0, 5);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+
+ try {
+ t.subSequence(5, 0);
+ fail("Should throw IndexOutOfBoundsException");
+ } catch(IndexOutOfBoundsException iobe) {
+ }
+ }
+
+ /*
+
+ // test speed of the dynamic instanceof checks in append(CharSequence),
+ // to find the best max length for the generic while (start<end) loop:
+ public void testAppendPerf() {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ final int count = 32;
+ CharSequence[] csq = new CharSequence[count * 6];
+ final StringBuilder sb = new StringBuilder();
+ for (int i=0,j=0; i<count; i++) {
+ sb.append(i%10);
+ final String testString = sb.toString();
+ CharTermAttribute cta = new CharTermAttributeImpl();
+ cta.append(testString);
+ csq[j++] = cta;
+ csq[j++] = testString;
+ csq[j++] = new StringBuilder(sb);
+ csq[j++] = new StringBuffer(sb);
+ csq[j++] = CharBuffer.wrap(testString.toCharArray());
+ csq[j++] = new CharSequence() {
+ public char charAt(int i) { return testString.charAt(i); }
+ public int length() { return testString.length(); }
+ public CharSequence subSequence(int start, int end) { return testString.subSequence(start, end); }
+ public String toString() { return testString; }
+ };
+ }
+
+ Random rnd = newRandom();
+ long startTime = System.currentTimeMillis();
+ for (int i=0; i<100000000; i++) {
+ t.setEmpty().append(csq[rnd.nextInt(csq.length)]);
+ }
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time: " + (endTime-startTime)/1000.0 + " s");
}
+ */
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java Tue May 4 11:18:46 2010
@@ -22,6 +22,7 @@ import org.apache.lucene.util.AttributeI
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
+@Deprecated
public class TestSimpleAttributeImpls extends LuceneTestCase {
public TestSimpleAttributeImpls(String name) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java Tue May 4 11:18:46 2010
@@ -107,10 +107,10 @@ public class TestTermAttributeImpl exten
char[] b = {'a', 'l', 'o', 'h', 'a'};
TermAttributeImpl t = new TermAttributeImpl();
t.setTermBuffer(b, 0, 5);
- assertEquals("term=aloha", t.toString());
+ assertEquals("aloha", t.toString());
t.setTermBuffer("hi there");
- assertEquals("term=hi there", t.toString());
+ assertEquals("hi there", t.toString());
}
public void testMixedStringArray() throws Exception {
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1,3 +1,5 @@
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932163,932369,932698,932747,932749,932773,932862,935521,940451
+/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:924791,924850
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880793,896906
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1,3 +1,5 @@
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932163,932369,932698,932747,932749,932773,932862,935521,940451
+/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:924791,924850
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880793,896906
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1,3 +1,5 @@
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932163,932369,932698,932747,932749,932773,932862,935521,940451
+/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924791,924850
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880793,896906
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Tue May 4 11:18:46 2010
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.Whites
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
@@ -152,15 +152,15 @@ public class TestDocumentWriter extends
restoreState(state);
payloadAtt.setPayload(null);
posIncrAtt.setPositionIncrement(0);
- termAtt.setTermBuffer(new char[]{'b'}, 0, 1);
+ termAtt.setEmpty().append("b");
state = null;
return true;
}
boolean hasNext = input.incrementToken();
if (!hasNext) return false;
- if (Character.isDigit(termAtt.termBuffer()[0])) {
- posIncrAtt.setPositionIncrement(termAtt.termBuffer()[0] - '0');
+ if (Character.isDigit(termAtt.buffer()[0])) {
+ posIncrAtt.setPositionIncrement(termAtt.buffer()[0] - '0');
}
if (first) {
// set payload on first position only
@@ -174,7 +174,7 @@ public class TestDocumentWriter extends
}
- TermAttribute termAtt = addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
};
@@ -215,7 +215,7 @@ public class TestDocumentWriter extends
private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
private int index = 0;
- private TermAttribute termAtt = addAttribute(TermAttribute.class);
+ private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@Override
public boolean incrementToken() throws IOException {
@@ -223,7 +223,7 @@ public class TestDocumentWriter extends
return false;
} else {
clearAttributes();
- termAtt.setTermBuffer(tokens[index++]);
+ termAtt.setEmpty().append(tokens[index++]);
return true;
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue May 4 11:18:46 2010
@@ -44,7 +44,7 @@ import org.apache.lucene.analysis.Whites
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -3479,7 +3479,7 @@ public class TestIndexWriter extends Luc
// LUCENE-1255
public void testNegativePositions() throws Throwable {
final TokenStream tokens = new TokenStream() {
- final TermAttribute termAtt = addAttribute(TermAttribute.class);
+ final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
final Iterator<String> terms = Arrays.asList("a","b","c").iterator();
@@ -3489,7 +3489,7 @@ public class TestIndexWriter extends Luc
public boolean incrementToken() {
if (!terms.hasNext()) return false;
clearAttributes();
- termAtt.setTermBuffer( terms.next());
+ termAtt.append(terms.next());
posIncrAtt.setPositionIncrement(first ? 0 : 1);
first = false;
return true;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java Tue May 4 11:18:46 2010
@@ -33,7 +33,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -523,7 +523,7 @@ public class TestPayloads extends Lucene
private ByteArrayPool pool;
private String term;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
PayloadAttribute payloadAtt;
PoolingPayloadTokenStream(ByteArrayPool pool) {
@@ -533,7 +533,7 @@ public class TestPayloads extends Lucene
term = pool.bytesToString(payload);
first = true;
payloadAtt = addAttribute(PayloadAttribute.class);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
}
@Override
@@ -541,7 +541,7 @@ public class TestPayloads extends Lucene
if (!first) return false;
first = false;
clearAttributes();
- termAtt.setTermBuffer(term);
+ termAtt.append(term);
payloadAtt.setPayload(new Payload(payload));
return true;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Tue May 4 11:18:46 2010
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.MockRAMDirectory;
@@ -123,12 +123,12 @@ public class TestTermVectorsReader exten
private class MyTokenStream extends TokenStream {
int tokenUpto;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
OffsetAttribute offsetAtt;
public MyTokenStream() {
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@@ -140,7 +140,7 @@ public class TestTermVectorsReader exten
else {
final TestToken testToken = tokens[tokenUpto++];
clearAttributes();
- termAtt.setTermBuffer(testToken.text);
+ termAtt.append(testToken.text);
offsetAtt.setOffset(testToken.startOffset, testToken.endOffset);
if (tokenUpto > 1) {
posIncrAtt.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java Tue May 4 11:18:46 2010
@@ -23,7 +23,7 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -33,12 +33,12 @@ import org.apache.lucene.util.LuceneTest
class RepeatingTokenStream extends TokenStream {
public int num;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
String value;
public RepeatingTokenStream(String val) {
this.value = val;
- this.termAtt = addAttribute(TermAttribute.class);
+ this.termAtt = addAttribute(CharTermAttribute.class);
}
@Override
@@ -46,7 +46,7 @@ class RepeatingTokenStream extends Token
num--;
if (num >= 0) {
clearAttributes();
- termAtt.setTermBuffer(value);
+ termAtt.append(value);
return true;
}
return false;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Tue May 4 11:18:46 2010
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.search.Query;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -148,14 +148,14 @@ public class TestMultiAnalyzer extends B
private int prevStartOffset;
private int prevEndOffset;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
OffsetAttribute offsetAtt;
TypeAttribute typeAtt;
public TestFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
@@ -164,7 +164,7 @@ public class TestMultiAnalyzer extends B
@Override
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
- termAtt.setTermBuffer("multi"+(multiToken+1));
+ termAtt.setEmpty().append("multi"+(multiToken+1));
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
typeAtt.setType(prevType);
posIncrAtt.setPositionIncrement(0);
@@ -178,7 +178,7 @@ public class TestMultiAnalyzer extends B
prevType = typeAtt.type();
prevStartOffset = offsetAtt.startOffset();
prevEndOffset = offsetAtt.endOffset();
- String text = termAtt.term();
+ String text = termAtt.toString();
if (text.equals("triplemulti")) {
multiToken = 2;
return true;
@@ -212,21 +212,21 @@ public class TestMultiAnalyzer extends B
private final class TestPosIncrementFilter extends TokenFilter {
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
public TestPosIncrementFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
public final boolean incrementToken () throws java.io.IOException {
while(input.incrementToken()) {
- if (termAtt.term().equals("the")) {
+ if (termAtt.toString().equals("the")) {
// stopword, do nothing
- } else if (termAtt.term().equals("quick")) {
+ } else if (termAtt.toString().equals("quick")) {
posIncrAtt.setPositionIncrement(2);
return true;
} else {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java Tue May 4 11:18:46 2010
@@ -40,7 +40,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
@@ -82,7 +82,7 @@ public class TestQueryParser extends Loc
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
public static class QPTestFilter extends TokenFilter {
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
OffsetAttribute offsetAtt;
/**
@@ -91,7 +91,7 @@ public class TestQueryParser extends Loc
*/
public QPTestFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@@ -103,19 +103,19 @@ public class TestQueryParser extends Loc
if (inPhrase) {
inPhrase = false;
clearAttributes();
- termAtt.setTermBuffer("phrase2");
+ termAtt.append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while (input.incrementToken()) {
- if (termAtt.term().equals("phrase")) {
+ if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
- termAtt.setTermBuffer("phrase1");
+ termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
- } else if (!termAtt.term().equals("stop"))
+ } else if (!termAtt.toString().equals("stop"))
return true;
}
return false;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java Tue May 4 11:18:46 2010
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.Whites
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -71,7 +71,7 @@ public class TestPositionIncrement exten
private int i = 0;
PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- TermAttribute termAtt = addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
@@ -79,7 +79,7 @@ public class TestPositionIncrement exten
if (i == TOKENS.length)
return false;
clearAttributes();
- termAtt.setTermBuffer(TOKENS[i]);
+ termAtt.append(TOKENS[i]);
offsetAtt.setOffset(i,i);
posIncrAtt.setPositionIncrement(INCREMENTS[i]);
i++;
@@ -331,7 +331,7 @@ class PayloadFilter extends TokenFilter
final PositionIncrementAttribute posIncrAttr;
final PayloadAttribute payloadAttr;
- final TermAttribute termAttr;
+ final CharTermAttribute termAttr;
public PayloadFilter(TokenStream input, String fieldName) {
super(input);
@@ -340,7 +340,7 @@ class PayloadFilter extends TokenFilter
i = 0;
posIncrAttr = input.addAttribute(PositionIncrementAttribute.class);
payloadAttr = input.addAttribute(PayloadAttribute.class);
- termAttr = input.addAttribute(TermAttribute.class);
+ termAttr = input.addAttribute(CharTermAttribute.class);
}
@Override
@@ -355,7 +355,9 @@ class PayloadFilter extends TokenFilter
}
posIncrAttr.setPositionIncrement(posIncr);
pos += posIncr;
- // System.out.println("term=" + termAttr.term() + " pos=" + pos);
+ if (TestPositionIncrement.VERBOSE) {
+ System.out.println("term=" + termAttr + " pos=" + pos);
+ }
i++;
return true;
} else {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java Tue May 4 11:18:46 2010
@@ -27,7 +27,7 @@ import org.apache.lucene.store.RAMDirect
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
@@ -261,27 +261,25 @@ public class TestTermRangeQuery extends
private static class SingleCharTokenizer extends Tokenizer {
char[] buffer = new char[1];
- boolean done;
- TermAttribute termAtt;
+ boolean done = false;
+ CharTermAttribute termAtt;
public SingleCharTokenizer(Reader r) {
super(r);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
- int count = input.read(buffer);
if (done)
return false;
else {
+ int count = input.read(buffer);
clearAttributes();
done = true;
if (count == 1) {
- termAtt.termBuffer()[0] = buffer[0];
- termAtt.setTermLength(1);
- } else
- termAtt.setTermLength(0);
+ termAtt.copyBuffer(buffer, 0, 1);
+ }
return true;
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Tue May 4 11:18:46 2010
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
@@ -479,7 +479,7 @@ public class TestPayloadSpans extends Lu
Set<String> nopayload = new HashSet<String>();
int pos;
PayloadAttribute payloadAtt;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
public PayloadFilter(TokenStream input, String fieldName) {
@@ -490,7 +490,7 @@ public class TestPayloadSpans extends Lu
entities.add("one");
nopayload.add("nopayload");
nopayload.add("np");
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
}
@@ -498,7 +498,7 @@ public class TestPayloadSpans extends Lu
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- String token = new String(termAtt.termBuffer(), 0, termAtt.termLength());
+ String token = termAtt.toString();
if (!nopayload.contains(token)) {
if (entities.contains(token)) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java Tue May 4 11:18:46 2010
@@ -27,27 +27,27 @@ public class TestAttributeSource extends
public void testCaptureState() {
// init a first instance
AttributeSource src = new AttributeSource();
- TermAttribute termAtt = src.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = src.addAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
- termAtt.setTermBuffer("TestTerm");
+ termAtt.append("TestTerm");
typeAtt.setType("TestType");
final int hashCode = src.hashCode();
AttributeSource.State state = src.captureState();
// modify the attributes
- termAtt.setTermBuffer("AnotherTestTerm");
+ termAtt.setEmpty().append("AnotherTestTerm");
typeAtt.setType("AnotherTestType");
assertTrue("Hash code should be different", hashCode != src.hashCode());
src.restoreState(state);
- assertEquals("TestTerm", termAtt.term());
+ assertEquals("TestTerm", termAtt.toString());
assertEquals("TestType", typeAtt.type());
assertEquals("Hash code should be equal after restore", hashCode, src.hashCode());
// restore into an exact configured copy
AttributeSource copy = new AttributeSource();
- copy.addAttribute(TermAttribute.class);
+ copy.addAttribute(CharTermAttribute.class);
copy.addAttribute(TypeAttribute.class);
copy.restoreState(state);
assertEquals("Both AttributeSources should have same hashCode after restore", src.hashCode(), copy.hashCode());
@@ -57,17 +57,17 @@ public class TestAttributeSource extends
AttributeSource src2 = new AttributeSource();
typeAtt = src2.addAttribute(TypeAttribute.class);
FlagsAttribute flagsAtt = src2.addAttribute(FlagsAttribute.class);
- termAtt = src2.addAttribute(TermAttribute.class);
+ termAtt = src2.addAttribute(CharTermAttribute.class);
flagsAtt.setFlags(12345);
src2.restoreState(state);
- assertEquals("TestTerm", termAtt.term());
+ assertEquals("TestTerm", termAtt.toString());
assertEquals("TestType", typeAtt.type());
assertEquals("FlagsAttribute should not be touched", 12345, flagsAtt.getFlags());
// init a third instance missing one Attribute
AttributeSource src3 = new AttributeSource();
- termAtt = src3.addAttribute(TermAttribute.class);
+ termAtt = src3.addAttribute(CharTermAttribute.class);
try {
src3.restoreState(state);
fail("The third instance is missing the TypeAttribute, so restoreState() should throw IllegalArgumentException");
@@ -78,42 +78,42 @@ public class TestAttributeSource extends
public void testCloneAttributes() {
final AttributeSource src = new AttributeSource();
- final TermAttribute termAtt = src.addAttribute(TermAttribute.class);
+ final FlagsAttribute flagsAtt = src.addAttribute(FlagsAttribute.class);
final TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
- termAtt.setTermBuffer("TestTerm");
+ flagsAtt.setFlags(1234);
typeAtt.setType("TestType");
final AttributeSource clone = src.cloneAttributes();
final Iterator<Class<? extends Attribute>> it = clone.getAttributeClassesIterator();
- assertEquals("TermAttribute must be the first attribute", TermAttribute.class, it.next());
+ assertEquals("FlagsAttribute must be the first attribute", FlagsAttribute.class, it.next());
assertEquals("TypeAttribute must be the second attribute", TypeAttribute.class, it.next());
assertFalse("No more attributes", it.hasNext());
- final TermAttribute termAtt2 = clone.getAttribute(TermAttribute.class);
+ final FlagsAttribute flagsAtt2 = clone.getAttribute(FlagsAttribute.class);
final TypeAttribute typeAtt2 = clone.getAttribute(TypeAttribute.class);
- assertNotSame("TermAttribute of original and clone must be different instances", termAtt2, termAtt);
+ assertNotSame("FlagsAttribute of original and clone must be different instances", flagsAtt2, flagsAtt);
assertNotSame("TypeAttribute of original and clone must be different instances", typeAtt2, typeAtt);
- assertEquals("TermAttribute of original and clone must be equal", termAtt2, termAtt);
+ assertEquals("FlagsAttribute of original and clone must be equal", flagsAtt2, flagsAtt);
assertEquals("TypeAttribute of original and clone must be equal", typeAtt2, typeAtt);
// test copy back
- termAtt2.setTermBuffer("OtherTerm");
+ flagsAtt2.setFlags(4711);
typeAtt2.setType("OtherType");
clone.copyTo(src);
- assertEquals("TermAttribute of original must now contain updated term", "OtherTerm", termAtt.term());
+ assertEquals("FlagsAttribute of original must now contain updated term", 4711, flagsAtt.getFlags());
assertEquals("TypeAttribute of original must now contain updated type", "OtherType", typeAtt.type());
// verify again:
- assertNotSame("TermAttribute of original and clone must be different instances", termAtt2, termAtt);
+ assertNotSame("FlagsAttribute of original and clone must be different instances", flagsAtt2, flagsAtt);
assertNotSame("TypeAttribute of original and clone must be different instances", typeAtt2, typeAtt);
- assertEquals("TermAttribute of original and clone must be equal", termAtt2, termAtt);
+ assertEquals("FlagsAttribute of original and clone must be equal", flagsAtt2, flagsAtt);
assertEquals("TypeAttribute of original and clone must be equal", typeAtt2, typeAtt);
}
public void testToStringAndMultiAttributeImplementations() {
AttributeSource src = new AttributeSource();
- TermAttribute termAtt = src.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = src.addAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
- termAtt.setTermBuffer("TestTerm");
+ termAtt.append("TestTerm");
typeAtt.setType("TestType");
assertEquals("Attributes should appear in original order", "("+termAtt.toString()+","+typeAtt.toString()+")", src.toString());
Iterator<AttributeImpl> it = src.getAttributeImplsIterator();
@@ -125,23 +125,23 @@ public class TestAttributeSource extends
src = new AttributeSource();
src.addAttributeImpl(new Token());
- // this should not add a new attribute as Token implements TermAttribute, too
- termAtt = src.addAttribute(TermAttribute.class);
- assertTrue("TermAttribute should be implemented by Token", termAtt instanceof Token);
+ // this should not add a new attribute as Token implements CharTermAttribute, too
+ termAtt = src.addAttribute(CharTermAttribute.class);
+ assertTrue("CharTermAttribute should be implemented by Token", termAtt instanceof Token);
// get the Token attribute and check, that it is the only one
it = src.getAttributeImplsIterator();
Token tok = (Token) it.next();
assertFalse("There should be only one attribute implementation instance", it.hasNext());
- termAtt.setTermBuffer("TestTerm");
+ termAtt.setEmpty().append("TestTerm");
assertEquals("Token should only printed once", "("+tok.toString()+")", src.toString());
}
public void testDefaultAttributeFactory() throws Exception {
AttributeSource src = new AttributeSource();
- assertTrue("TermAttribute is not implemented by TermAttributeImpl",
- src.addAttribute(TermAttribute.class) instanceof TermAttributeImpl);
+ assertTrue("CharTermAttribute is not implemented by CharTermAttributeImpl",
+ src.addAttribute(CharTermAttribute.class) instanceof CharTermAttributeImpl);
assertTrue("OffsetAttribute is not implemented by OffsetAttributeImpl",
src.addAttribute(OffsetAttribute.class) instanceof OffsetAttributeImpl);
assertTrue("FlagsAttribute is not implemented by FlagsAttributeImpl",
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1,2 +1,4 @@
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:932163,932369,932698,932747,932749,932773,932862,935521,940451
+/lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:924791,924850
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:896850,909334
/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-925561
Propchange: lucene/dev/branches/branch_3x/solr/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1,3 +1,4 @@
+/lucene/dev/trunk/solr:932163,932369,932698,932747,932749,932773,932862,935521,940451
/lucene/java/branches/lucene_2_4/solr:748824
/lucene/java/branches/lucene_2_9/solr:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr:818601-821336
Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Tue May 4 11:18:46 2010
@@ -59,7 +59,8 @@ Upgrading from Solr 1.4
"terms" container is a map instead of a named list. This affects
response formats like JSON, but not XML. (yonik)
-
+* SOLR-1876: All Analyzers and TokenStreams are now final to enforce
+ the decorator pattern. (rmuir, uschindler)
Detailed Change List
----------------------
Modified: lucene/dev/branches/branch_3x/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/build.xml?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/build.xml Tue May 4 11:18:46 2010
@@ -455,6 +455,10 @@
<formatter classname="${junit.details.formatter}" usefile="false" if="junit.details"/>
<classpath refid="test.run.classpath"/>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ <enable package="org.apache.solr"/>
+ </assertions>
<formatter type="${junit.formatter}"/>
<batchtest fork="yes" todir="${junit.output.dir}" if="runall">
<fileset dir="src/test" includes="**/Test@{pattern}*.java,**/@{pattern}*Test.java"/>
Modified: lucene/dev/branches/branch_3x/solr/contrib/clustering/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/clustering/build.xml?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/clustering/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/clustering/build.xml Tue May 4 11:18:46 2010
@@ -139,6 +139,10 @@
>
<formatter type="brief" usefile="false" if="junit.details"/>
<classpath refid="test.classpath"/>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ <enable package="org.apache.solr"/>
+ </assertions>
<formatter type="xml"/>
<batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">
<fileset dir="src/test/java" includes="${junit.includes}">
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/build.xml?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/build.xml Tue May 4 11:18:46 2010
@@ -162,6 +162,10 @@
>
<formatter type="brief" usefile="false" if="junit.details"/>
<classpath refid="test.extras.classpath"/>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ <enable package="org.apache.solr"/>
+ </assertions>
<formatter type="xml"/>
<batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">
<fileset dir="src/extras/test/java" includes="${junit.includes}"/>
Modified: lucene/dev/branches/branch_3x/solr/contrib/extraction/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/extraction/build.xml?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/extraction/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/extraction/build.xml Tue May 4 11:18:46 2010
@@ -88,6 +88,10 @@
>
<formatter type="brief" usefile="false" if="junit.details"/>
<classpath refid="test.classpath"/>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ <enable package="org.apache.solr"/>
+ </assertions>
<formatter type="xml"/>
<batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">
<fileset dir="src/test/java" includes="${junit.includes}"/>
Modified: lucene/dev/branches/branch_3x/solr/contrib/velocity/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/velocity/build.xml?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/velocity/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/velocity/build.xml Tue May 4 11:18:46 2010
@@ -87,6 +87,10 @@
<sysproperty key="java.util.logging.config.file" value="${common-solr.dir}/testlogging.properties"/>
<formatter type="brief" usefile="false"/>
<classpath refid="test.classpath"/>
+ <assertions>
+ <enable package="org.apache.lucene"/>
+ <enable package="org.apache.solr"/>
+ </assertions>
<!--<formatter type="xml" usefile="false"/>-->
<batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">
<fileset dir="src/test" includes="${junit.includes}"/>
Propchange: lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1 +1,2 @@
+/lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar:932163,932369,932698,932747,932749,932773,932862,935521,940451
/lucene/solr/trunk/lib/commons-httpclient-3.1.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1 +1,2 @@
+/lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar:932163,932369,932698,932747,932749,932773,932862,935521,940451
/lucene/solr/trunk/lib/jcl-over-slf4j-1.5.5.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 4 11:18:46 2010
@@ -1 +1,2 @@
+/lucene/dev/trunk/solr/src/common/org/apache/solr/common:932163,932369,932698,932747,932749,932773,932862,935521,940451
/lucene/solr/trunk/src/common/org/apache/solr/common:922950-923910,923912-925091
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java Tue May 4 11:18:46 2010
@@ -20,11 +20,11 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource; // javadoc @link
@@ -73,7 +73,7 @@ public abstract class BufferedTokenStrea
private final LinkedList<Token> inQueue = new LinkedList<Token>();
private final LinkedList<Token> outQueue = new LinkedList<Token>();
- private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
@@ -150,7 +150,7 @@ public abstract class BufferedTokenStrea
return null;
} else {
Token token = new Token();
- token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
+ token.setTermBuffer(termAtt.buffer(), 0, termAtt.length());
token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
token.setType(typeAtt.type());
token.setFlags(flagsAtt.getFlags());
@@ -163,7 +163,7 @@ public abstract class BufferedTokenStrea
/** old api emulation for back compat */
private boolean writeToken(Token token) throws IOException {
clearAttributes();
- termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
+ termAtt.copyBuffer(token.termBuffer(), 0, token.termLength());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
typeAtt.setType(token.type());
flagsAtt.setFlags(token.getFlags());
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java Tue May 4 11:18:46 2010
@@ -18,7 +18,7 @@
package org.apache.solr.analysis;
import org.apache.lucene.analysis.*;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
import java.util.ArrayList;
@@ -188,22 +188,21 @@ public class CapitalizationFilterFactory
* <p/>
* This is package protected since it is not useful without the Factory
*/
-class CapitalizationFilter extends TokenFilter {
+final class CapitalizationFilter extends TokenFilter {
private final CapitalizationFilterFactory factory;
- private final TermAttribute termAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public CapitalizationFilter(TokenStream in, final CapitalizationFilterFactory factory) {
super(in);
this.factory = factory;
- this.termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
- char[] termBuffer = termAtt.termBuffer();
- int termBufferLength = termAtt.termLength();
+ char[] termBuffer = termAtt.buffer();
+ int termBufferLength = termAtt.length();
char[] backup = null;
if (factory.maxWordCount < CapitalizationFilterFactory.DEFAULT_MAX_WORD_COUNT) {
//make a backup in case we exceed the word count
@@ -232,7 +231,7 @@ class CapitalizationFilter extends Token
}
if (wordCount > factory.maxWordCount) {
- termAtt.setTermBuffer(backup, 0, termBufferLength);
+ termAtt.copyBuffer(backup, 0, termBufferLength);
}
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CommonGramsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CommonGramsFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CommonGramsFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/CommonGramsFilter.java Tue May 4 11:18:46 2010
@@ -18,7 +18,7 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.Version;
@@ -52,7 +52,7 @@ public final class CommonGramsFilter ext
private final StringBuilder buffer = new StringBuilder();
- private final TermAttribute termAttribute = addAttribute(TermAttribute.class);
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
@@ -231,7 +231,7 @@ public final class CommonGramsFilter ext
* @return {@code true} if the current token is a common term, {@code false} otherwise
*/
private boolean isCommon() {
- return commonWords != null && commonWords.contains(termAttribute.termBuffer(), 0, termAttribute.termLength());
+ return commonWords != null && commonWords.contains(termAttribute.buffer(), 0, termAttribute.length());
}
/**
@@ -239,7 +239,7 @@ public final class CommonGramsFilter ext
*/
private void saveTermBuffer() {
buffer.setLength(0);
- buffer.append(termAttribute.termBuffer(), 0, termAttribute.termLength());
+ buffer.append(termAttribute.buffer(), 0, termAttribute.length());
buffer.append(SEPARATOR);
lastStartOffset = offsetAttribute.startOffset();
lastWasCommon = isCommon();
@@ -249,19 +249,19 @@ public final class CommonGramsFilter ext
* Constructs a compound token.
*/
private void gramToken() {
- buffer.append(termAttribute.termBuffer(), 0, termAttribute.termLength());
+ buffer.append(termAttribute.buffer(), 0, termAttribute.length());
int endOffset = offsetAttribute.endOffset();
clearAttributes();
int length = buffer.length();
- char termText[] = termAttribute.termBuffer();
+ char termText[] = termAttribute.buffer();
if (length > termText.length) {
- termText = termAttribute.resizeTermBuffer(length);
+ termText = termAttribute.resizeBuffer(length);
}
buffer.getChars(0, length, termText, 0);
- termAttribute.setTermLength(length);
+ termAttribute.setLength(length);
posIncAttribute.setPositionIncrement(0);
offsetAttribute.setOffset(lastStartOffset, endOffset);
typeAttribute.setType(GRAM_TYPE);
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/DoubleMetaphoneFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/DoubleMetaphoneFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/DoubleMetaphoneFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/DoubleMetaphoneFilter.java Tue May 4 11:18:46 2010
@@ -22,25 +22,23 @@ import java.util.LinkedList;
import org.apache.commons.codec.language.DoubleMetaphone;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-public class DoubleMetaphoneFilter extends TokenFilter {
+public final class DoubleMetaphoneFilter extends TokenFilter {
private static final String TOKEN_TYPE = "DoubleMetaphone";
private final LinkedList<State> remainingTokens = new LinkedList<State>();
private final DoubleMetaphone encoder = new DoubleMetaphone();
private final boolean inject;
- private final TermAttribute termAtt;
- private final PositionIncrementAttribute posAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
protected DoubleMetaphoneFilter(TokenStream input, int maxCodeLength, boolean inject) {
super(input);
this.encoder.setMaxCodeLen(maxCodeLength);
this.inject = inject;
- this.termAtt = addAttribute(TermAttribute.class);
- this.posAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
@@ -55,12 +53,12 @@ public class DoubleMetaphoneFilter exten
if (!input.incrementToken()) return false;
- int len = termAtt.termLength();
+ int len = termAtt.length();
if (len==0) return true; // pass through zero length terms
int firstAlternativeIncrement = inject ? 0 : posAtt.getPositionIncrement();
- String v = new String(termAtt.termBuffer(), 0, len);
+ String v = termAtt.toString();
String primaryPhoneticValue = encoder.doubleMetaphone(v);
String alternatePhoneticValue = encoder.doubleMetaphone(v, true);
@@ -74,7 +72,7 @@ public class DoubleMetaphoneFilter exten
}
posAtt.setPositionIncrement( firstAlternativeIncrement );
firstAlternativeIncrement = 0;
- termAtt.setTermBuffer(primaryPhoneticValue);
+ termAtt.setEmpty().append(primaryPhoneticValue);
saveState = true;
}
@@ -86,7 +84,7 @@ public class DoubleMetaphoneFilter exten
saveState = false;
}
posAtt.setPositionIncrement( firstAlternativeIncrement );
- termAtt.setTermBuffer(alternatePhoneticValue);
+ termAtt.setEmpty().append(alternatePhoneticValue);
saveState = true;
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java Tue May 4 11:18:46 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* When the plain text is extracted from documents, we will often have many words hyphenated and broken into
@@ -54,7 +54,7 @@ import org.apache.lucene.analysis.tokena
*/
public final class HyphenatedWordsFilter extends TokenFilter {
- private final TermAttribute termAttribute = addAttribute(TermAttribute.class);
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final StringBuilder hyphenated = new StringBuilder();
@@ -75,8 +75,8 @@ public final class HyphenatedWordsFilter
@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
- char[] term = termAttribute.termBuffer();
- int termLength = termAttribute.termLength();
+ char[] term = termAttribute.buffer();
+ int termLength = termAttribute.length();
if (termLength > 0 && term[termLength - 1] == '-') {
// a hyphenated word
@@ -128,14 +128,14 @@ public final class HyphenatedWordsFilter
restoreState(savedState);
savedState = null;
- char term[] = termAttribute.termBuffer();
+ char term[] = termAttribute.buffer();
int length = hyphenated.length();
- if (length > termAttribute.termLength()) {
- term = termAttribute.resizeTermBuffer(length);
+ if (length > termAttribute.length()) {
+ term = termAttribute.resizeBuffer(length);
}
hyphenated.getChars(0, length, term, 0);
- termAttribute.setTermLength(length);
+ termAttribute.setLength(length);
offsetAttribute.setOffset(offsetAttribute.startOffset(), endOffset);
hyphenated.setLength(0);
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/KeepWordFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/KeepWordFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/KeepWordFilter.java Tue May 4 11:18:46 2010
@@ -20,7 +20,7 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
import java.util.Set;
@@ -34,7 +34,7 @@ import java.util.Set;
*/
public final class KeepWordFilter extends TokenFilter {
private final CharArraySet words;
- private final TermAttribute termAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/** @deprecated Use {@link #KeepWordFilter(TokenStream, Set, boolean)} instead */
@Deprecated
@@ -47,13 +47,12 @@ public final class KeepWordFilter extend
public KeepWordFilter(TokenStream in, CharArraySet words) {
super(in);
this.words = words;
- this.termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
- if (words.contains(termAtt.termBuffer(), 0, termAtt.termLength())) return true;
+ if (words.contains(termAtt.buffer(), 0, termAtt.length())) return true;
}
return false;
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/PatternReplaceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/PatternReplaceFilter.java?rev=940806&r1=940805&r2=940806&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/PatternReplaceFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/PatternReplaceFilter.java Tue May 4 11:18:46 2010
@@ -19,12 +19,11 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.IOException;
-import java.nio.CharBuffer;
/**
* A TokenFilter which applies a Pattern to each token in the stream,
@@ -43,7 +42,9 @@ public final class PatternReplaceFilter
private final Pattern p;
private final String replacement;
private final boolean all;
- private final TermAttribute termAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final Matcher m;
+
/**
* Constructs an instance to replace either the first, or all occurances
*
@@ -63,20 +64,18 @@ public final class PatternReplaceFilter
this.p=p;
this.replacement = (null == replacement) ? "" : replacement;
this.all=all;
- this.termAtt = addAttribute(TermAttribute.class);
+ this.m = p.matcher(termAtt);
}
@Override
public boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
- CharSequence text = CharBuffer.wrap(termAtt.termBuffer(), 0, termAtt.termLength());
- Matcher m = p.matcher(text);
-
- if (all) {
- termAtt.setTermBuffer(m.replaceAll(replacement));
- } else {
- termAtt.setTermBuffer(m.replaceFirst(replacement));
+ m.reset();
+ if (m.find()) {
+ // replaceAll/replaceFirst will reset() this previous find.
+ String transformed = all ? m.replaceAll(replacement) : m.replaceFirst(replacement);
+ termAtt.setEmpty().append(transformed);
}
return true;