You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2009/08/22 14:40:09 UTC
svn commit: r806847 - in /lucene/java/trunk/src:
java/org/apache/lucene/analysis/ java/org/apache/lucene/util/
test/org/apache/lucene/analysis/ test/org/apache/lucene/index/
Author: uschindler
Date: Sat Aug 22 12:40:09 2009
New Revision: 806847
URL: http://svn.apache.org/viewvc?rev=806847&view=rev
Log:
LUCENE-1843: Convert some tests to new TokenStream API, better support of cross-impl AttributeImpl.copyTo()
Modified:
lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java
lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java
lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java Sat Aug 22 12:40:09 2009
@@ -859,11 +859,20 @@
}
public void copyTo(AttributeImpl target) {
- Token to = (Token) target;
- to.reinit(this);
- // reinit shares the payload, so clone it:
- if (payload !=null) {
- to.payload = (Payload) payload.clone();
+ if (target instanceof Token) {
+ final Token to = (Token) target;
+ to.reinit(this);
+ // reinit shares the payload, so clone it:
+ if (payload !=null) {
+ to.payload = (Payload) payload.clone();
+ }
+ } else {
+ ((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength);
+ ((OffsetAttribute) target).setOffset(startOffset, endOffset);
+ ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
+ ((PayloadAttribute) target).setPayload((payload == null) ? null : (Payload) payload.clone());
+ ((FlagsAttribute) target).setFlags(flags);
+ ((TypeAttribute) target).setType(type);
}
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java Sat Aug 22 12:40:09 2009
@@ -157,6 +157,10 @@
}
public void copyTo(AttributeImpl target) {
- ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
+ if (target instanceof TokenWrapper) {
+ ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
+ } else {
+ this.delegate.copyTo(target);
+ }
}
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java Sat Aug 22 12:40:09 2009
@@ -101,8 +101,8 @@
/**
* Copies the values from this Attribute into the passed-in
- * target attribute. The type of the target must match the type
- * of this attribute.
+ * target attribute. The target implementation must support all the
+ * Attributes this implementation supports.
*/
public abstract void copyTo(AttributeImpl target);
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java Sat Aug 22 12:40:09 2009
@@ -19,35 +19,25 @@
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public abstract class BaseTokenTestCase extends BaseTokenStreamTestCase {
-/* TODO: Convert to new TokenStream API. Token instances must be removed for that to work */
-public abstract class BaseTokenTestCase extends LuceneTestCase {
public static String tsToString(TokenStream in) throws IOException {
- StringBuffer out = new StringBuffer();
- Token t = in.next();
- if (null != t)
- out.append(new String(t.termBuffer(), 0, t.termLength()));
-
- for (t = in.next(); null != t; t = in.next()) {
- out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
+ final TermAttribute termAtt = (TermAttribute) in.addAttribute(TermAttribute.class);
+ final StringBuffer out = new StringBuffer();
+ in.reset();
+ while (in.incrementToken()) {
+ if (out.length()>0) out.append(' ');
+ out.append(termAtt.term());
}
in.close();
return out.toString();
}
-/*
- public List<String> tok2str(Iterable<Token> tokLst) {
- ArrayList<String> lst = new ArrayList<String>();
- for ( Token t : tokLst ) {
- lst.add( new String(t.termBuffer(), 0, t.termLength()));
- }
- return lst;
- }
-*/
public void assertTokEqual(List/*<Token>*/ a, List/*<Token>*/ b) {
assertTokEq(a,b,false);
@@ -64,7 +54,7 @@
for (Iterator iter = a.iterator(); iter.hasNext();) {
Token tok = (Token)iter.next();
pos += tok.getPositionIncrement();
- if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
+ if (!tokAt(b, tok.term(), pos
, checkOff ? tok.startOffset() : -1
, checkOff ? tok.endOffset() : -1
))
@@ -79,7 +69,7 @@
for (Iterator iter = lst.iterator(); iter.hasNext();) {
Token tok = (Token)iter.next();
pos += tok.getPositionIncrement();
- if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
+ if (pos==tokPos && tok.term().equals(val)
&& (startOff==-1 || tok.startOffset()==startOff)
&& (endOff ==-1 || tok.endOffset() ==endOff )
)
@@ -146,41 +136,22 @@
static List/*<Token>*/ getTokens(TokenStream tstream) throws IOException {
List/*<Token>*/ tokens = new ArrayList/*<Token>*/();
- while (true) {
- Token t = tstream.next();
- if (t==null) break;
+ tstream.reset();
+ while (tstream.incrementToken()) {
+ final Token t = new Token();
+ for (Iterator it = tstream.getAttributeImplsIterator(); it.hasNext();) {
+ final AttributeImpl att = (AttributeImpl) it.next();
+ try {
+ att.copyTo(t);
+ } catch (ClassCastException ce) {
+ // ignore Attributes unsupported by Token
+ }
+ }
tokens.add(t);
}
+ tstream.close();
+
return tokens;
}
-/*
- public static class IterTokenStream extends TokenStream {
- Iterator<Token> toks;
- public IterTokenStream(Token... toks) {
- this.toks = Arrays.asList(toks).iterator();
- }
- public IterTokenStream(Iterable<Token> toks) {
- this.toks = toks.iterator();
- }
- public IterTokenStream(Iterator<Token> toks) {
- this.toks = toks;
- }
- public IterTokenStream(String ... text) {
- int off = 0;
- ArrayList<Token> t = new ArrayList<Token>( text.length );
- for( String txt : text ) {
- t.add( new Token( txt, off, off+txt.length() ) );
- off += txt.length() + 2;
- }
- this.toks = t.iterator();
- }
- @Override
- public Token next() {
- if (toks.hasNext()) {
- return toks.next();
- }
- return null;
- }
- }
-*/
+
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Aug 22 12:40:09 2009
@@ -29,21 +29,23 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.HashSet;
import java.util.Random;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.SimpleAnalyzer;
-import org.apache.lucene.analysis.SinkTokenizer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.TeeSinkTokenFilter;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
@@ -62,7 +64,6 @@
import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
@@ -70,8 +71,16 @@
*
* @version $Id$
*/
-public class TestIndexWriter extends LuceneTestCase
-{
+public class TestIndexWriter extends BaseTokenStreamTestCase {
+ public TestIndexWriter(String name) {
+ super(name, new HashSet(Arrays.asList(new String[]{
+ "testExceptionFromTokenStream", "testDocumentsWriterExceptions", "testNegativePositions",
+ "testEndOffsetPositionWithCachingTokenFilter", "testEndOffsetPositionWithTeeSinkTokenFilter",
+ "testEndOffsetPositionStandard", "testEndOffsetPositionStandardEmptyField",
+ "testEndOffsetPositionStandardEmptyField2"
+ })));
+ }
+
public void testDocCount() throws IOException
{
Directory dir = new RAMDirectory();
@@ -3530,16 +3539,22 @@
// LUCENE-1255
public void testNegativePositions() throws Throwable {
- SinkTokenizer tokens = new SinkTokenizer();
- Token t = new Token();
- t.setTermBuffer("a");
- t.setPositionIncrement(0);
- tokens.add(t);
- t.setTermBuffer("b");
- t.setPositionIncrement(1);
- tokens.add(t);
- t.setTermBuffer("c");
- tokens.add(t);
+ final TokenStream tokens = new TokenStream() {
+ final TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+ final PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+
+ final Iterator tokens = Arrays.asList(new String[]{"a","b","c"}).iterator();
+ boolean first = true;
+
+ public boolean incrementToken() {
+ if (!tokens.hasNext()) return false;
+ clearAttributes();
+ termAtt.setTermBuffer((String) tokens.next());
+ posIncrAtt.setPositionIncrement(first ? 0 : 1);
+ first = false;
+ return true;
+ }
+ };
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);