You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2009/08/22 14:04:37 UTC

svn commit: r806844 - in /lucene/java/trunk: contrib/highlighter/src/java/org/apache/lucene/search/highlight/ contrib/highlighter/src/test/org/apache/lucene/search/highlight/ contrib/memory/src/test/org/apache/lucene/index/memory/ src/java/org/apache/l...

Author: uschindler
Date: Sat Aug 22 12:04:35 2009
New Revision: 806844

URL: http://svn.apache.org/viewvc?rev=806844&view=rev
Log:
LUCENE-1825: Incorrect usage of AttributeSource.addAttribute/getAttribute leads to failures when onlyUseNewAPI=true

Added:
    lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java   (with props)
Modified:
    lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
    lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
    lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java
    lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
    lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
    lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
    lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
    lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java
    lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/TermsHashPerField.java
    lucene/java/trunk/src/java/org/apache/lucene/search/QueryTermVector.java
    lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
    lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
    lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java
    lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java
    lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java
    lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java

Modified: lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java Sat Aug 22 12:04:35 2009
@@ -156,8 +156,8 @@
    */
   public TokenStream init(TokenStream tokenStream) throws IOException {
     position = -1;
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-    posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
+    termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class);
+    posIncAtt = (PositionIncrementAttribute) tokenStream.addAttribute(PositionIncrementAttribute.class);
     if(!skipInitExtractor) {
       if(fieldWeightedSpanTerms != null) {
         fieldWeightedSpanTerms.clear();

Modified: lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java Sat Aug 22 12:04:35 2009
@@ -95,7 +95,7 @@
    * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
    */
   public TokenStream init(TokenStream tokenStream) {
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+    termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class);
     return null;
   }
 

Modified: lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java Sat Aug 22 12:04:35 2009
@@ -47,7 +47,7 @@
    * @see org.apache.lucene.search.highlight.Fragmenter#start(java.lang.String, org.apache.lucene.analysis.TokenStream)
    */
   public void start(String originalText, TokenStream stream) {
-    offsetAtt = (OffsetAttribute) stream.getAttribute(OffsetAttribute.class);
+    offsetAtt = (OffsetAttribute) stream.addAttribute(OffsetAttribute.class);
     currentNumFrags = 1;
   }
 

Modified: lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java Sat Aug 22 12:04:35 2009
@@ -101,8 +101,8 @@
     position = -1;
     currentNumFrags = 1;
     textSize = originalText.length();
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-    posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
-    offsetAtt = (OffsetAttribute) tokenStream.getAttribute(OffsetAttribute.class);
+    termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class);
+    posIncAtt = (PositionIncrementAttribute) tokenStream.addAttribute(PositionIncrementAttribute.class);
+    offsetAtt = (OffsetAttribute) tokenStream.addAttribute(OffsetAttribute.class);
   }
 }

Modified: lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Sat Aug 22 12:04:35 2009
@@ -41,8 +41,8 @@
   private TermAttribute termAtt;
 
   public TokenGroup(TokenStream tokenStream) {
-    offsetAtt = (OffsetAttribute) tokenStream.getAttribute(OffsetAttribute.class);
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+    offsetAtt = (OffsetAttribute) tokenStream.addAttribute(OffsetAttribute.class);
+    termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class);
   }
 
   void addToken(float score) {

Modified: lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Sat Aug 22 12:04:35 2009
@@ -33,8 +33,6 @@
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
-import junit.framework.TestCase;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.LowerCaseTokenizer;
 import org.apache.lucene.analysis.SimpleAnalyzer;
@@ -78,6 +76,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.Version;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 
@@ -85,7 +84,7 @@
  * JUnit Test for Highlighter class.
  *
  */
-public class HighlighterTest extends TestCase implements Formatter {
+public class HighlighterTest extends BaseTokenStreamTestCase implements Formatter {
   private IndexReader reader;
   static final String FIELD_NAME = "contents";
   private Query query;
@@ -1600,10 +1599,8 @@
     }
   }
 
-  /*
-   * @see TestCase#setUp()
-   */
   protected void setUp() throws Exception {
+    super.setUp();
     ramDir = new RAMDirectory();
     IndexWriter writer = new IndexWriter(ramDir, new StandardAnalyzer(), true);
     for (int i = 0; i < texts.length; i++) {
@@ -1624,9 +1621,6 @@
 
   }
 
-  /*
-   * @see TestCase#tearDown()
-   */
   protected void tearDown() throws Exception {
     super.tearDown();
   }
@@ -1692,9 +1686,9 @@
   public SynonymTokenizer(TokenStream realStream, Map synonyms) {
     this.realStream = realStream;
     this.synonyms = synonyms;
-    realTermAtt = (TermAttribute) realStream.getAttribute(TermAttribute.class);
-    realPosIncrAtt = (PositionIncrementAttribute) realStream.getAttribute(PositionIncrementAttribute.class);
-    realOffsetAtt = (OffsetAttribute) realStream.getAttribute(OffsetAttribute.class);
+    realTermAtt = (TermAttribute) realStream.addAttribute(TermAttribute.class);
+    realPosIncrAtt = (PositionIncrementAttribute) realStream.addAttribute(PositionIncrementAttribute.class);
+    realOffsetAtt = (OffsetAttribute) realStream.addAttribute(OffsetAttribute.class);
 
     termAtt = (TermAttribute) addAttribute(TermAttribute.class);
     posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);

Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sat Aug 22 12:04:35 2009
@@ -33,8 +33,7 @@
 import java.util.List;
 import java.util.Set;
 
-import junit.framework.TestCase;
-
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.analysis.StopAnalyzer;
@@ -198,7 +197,7 @@
 </pre>
 
 */
-public class MemoryIndexTest extends TestCase {
+public class MemoryIndexTest extends BaseTokenStreamTestCase {
   
   private Analyzer analyzer;
   private boolean fastMode = false;
@@ -214,7 +213,8 @@
 
   /* all files will be open relative to this */
   public String fileDir;
-  public void setUp() {
+  protected void setUp() throws Exception {
+    super.setUp();
     fileDir = System.getProperty("lucene.common.dir", null);
   }
   

Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (original)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java Sat Aug 22 12:04:35 2009
@@ -31,8 +31,7 @@
 import java.util.Set;
 import java.util.regex.Pattern;
 
-import junit.framework.TestCase;
-
+import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.LetterTokenizer;
 import org.apache.lucene.analysis.LowerCaseFilter;
 import org.apache.lucene.analysis.StopAnalyzer;
@@ -58,8 +57,9 @@
 Thus the PatternAnalyzer produces correct output, whereas the WhitespaceAnalyzer 
 silently truncates text, and so the comparison results in assertEquals() don't match up. 
 
+TODO: Convert to new TokenStream API!
 */
-public class PatternAnalyzerTest extends TestCase {
+public class PatternAnalyzerTest extends LuceneTestCase {
   
   /** Runs the tests and/or benchmark */
   public static void main(String[] args) throws Throwable {

Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java (original)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java Sat Aug 22 12:04:35 2009
@@ -31,10 +31,9 @@
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 
-import junit.framework.TestCase;
-
-public class TestSynonymTokenFilter extends TestCase {
+public class TestSynonymTokenFilter extends BaseTokenStreamTestCase {
   File dataDir = new File(System.getProperty("dataDir", "./bin"));
   File testFile = new File(dataDir, "org/apache/lucene/index/memory/testSynonyms.txt");
   

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/TermsHashPerField.java Sat Aug 22 12:04:35 2009
@@ -249,7 +249,7 @@
   private boolean doNextCall;
 
   void start(Fieldable f) {
-    termAtt = (TermAttribute) fieldState.attributeSource.getAttribute(TermAttribute.class);
+    termAtt = (TermAttribute) fieldState.attributeSource.addAttribute(TermAttribute.class);
     consumer.start(f);
     if (nextPerField != null) {
       nextPerField.start(f);

Modified: lucene/java/trunk/src/java/org/apache/lucene/search/QueryTermVector.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/search/QueryTermVector.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/search/QueryTermVector.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/search/QueryTermVector.java Sat Aug 22 12:04:35 2009
@@ -61,7 +61,7 @@
           boolean hasMoreTokens = false;
           
           stream.reset(); 
-          TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
+          TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
 
           hasMoreTokens = stream.incrementToken();
           while (hasMoreTokens) {

Modified: lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java Sat Aug 22 12:04:35 2009
@@ -249,7 +249,11 @@
    * <p>Signature for Java 1.5: <code>public &lt;T extends Attribute&gt; T getAttribute(Class&lt;T&gt;)</code>
    * 
    * @throws IllegalArgumentException if this AttributeSource does not contain the
-   *         Attribute
+   *         Attribute. It is recommended to always use {@link #addAttribute} even in consumers
+   *         of TokenStreams, because you cannot know if a specific TokenStream really uses
+   *         a specific Attribute. {@link #addAttribute} will automatically make the attribute
+   *         available. If you want to only use the attribute, if it is available (to optimize
+   *         consuming), use {@link #hasAttribute}.
    */
   public Attribute getAttribute(Class attClass) {
     final Attribute att = (Attribute) this.attributes.get(attClass);

Added: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=806844&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sat Aug 22 12:04:35 2009
@@ -0,0 +1,84 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Set;
+ 
+import org.apache.lucene.util.LuceneTestCase;
+
+/** 
+ * Base class for all Lucene unit tests that use TokenStreams.  
+ * <p>
+ * This class runs all tests twice, one time with {@link TokenStream#setOnlyUseNewAPI} <code>false</code>
+ * and after that one time with <code>true</code>.
+ */
+public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
+
+  private boolean onlyUseNewAPI = false;
+  private final Set testWithNewAPI;
+  
+  public BaseTokenStreamTestCase() {
+    super();
+    this.testWithNewAPI = null; // run all tests also with onlyUseNewAPI
+  }
+
+  public BaseTokenStreamTestCase(String name) {
+    super(name);
+    this.testWithNewAPI = null; // run all tests also with onlyUseNewAPI
+  }
+
+  public BaseTokenStreamTestCase(Set testWithNewAPI) {
+    super();
+    this.testWithNewAPI = testWithNewAPI;
+  }
+
+  public BaseTokenStreamTestCase(String name, Set testWithNewAPI) {
+    super(name);
+    this.testWithNewAPI = testWithNewAPI;
+  }
+
+  // @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    TokenStream.setOnlyUseNewAPI(onlyUseNewAPI);
+  }
+
+  // @Override
+  public void runBare() throws Throwable {
+    // Do the test with onlyUseNewAPI=false (default)
+    try {
+      onlyUseNewAPI = false;
+      super.runBare();
+    } catch (Throwable e) {
+      System.out.println("Test failure of "+getName()+" occurred with onlyUseNewAPI=false");
+      throw e;
+    }
+
+    if (testWithNewAPI == null || testWithNewAPI.contains(getName())) {
+      // Do the test again with onlyUseNewAPI=true
+      try {
+        onlyUseNewAPI = true;
+        super.runBare();
+      } catch (Throwable e) {
+        System.out.println("Test failure of "+getName()+" occurred with onlyUseNewAPI=true");
+        throw e;
+      }
+    }
+  }
+
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java Sat Aug 22 12:04:35 2009
@@ -25,6 +25,7 @@
 
 import org.apache.lucene.util.LuceneTestCase;
 
+/* TODO: Convert to new TokenStream API. Token instances must be removed for that to work */
 public abstract class BaseTokenTestCase extends LuceneTestCase {
   public static String tsToString(TokenStream in) throws IOException {
     StringBuffer out = new StringBuffer();

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java Sat Aug 22 12:04:35 2009
@@ -18,14 +18,13 @@
  */
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Iterator;
 
-public class TestASCIIFoldingFilter extends LuceneTestCase {
+public class TestASCIIFoldingFilter extends BaseTokenStreamTestCase {
 
   // testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
   public void testLatin1Accents() throws Exception {

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Sat Aug 22 12:04:35 2009
@@ -26,9 +26,8 @@
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.Payload;
-import org.apache.lucene.util.LuceneTestCase;
 
-public class TestAnalyzers extends LuceneTestCase {
+public class TestAnalyzers extends BaseTokenStreamTestCase {
 
    public TestAnalyzers(String name) {
       super(name);

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sat Aug 22 12:04:35 2009
@@ -20,8 +20,6 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.util.LuceneTestCase;
-
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
@@ -34,7 +32,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
-public class TestCachingTokenFilter extends LuceneTestCase {
+public class TestCachingTokenFilter extends BaseTokenStreamTestCase {
   private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
   
   public void testCaching() throws IOException {

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Sat Aug 22 12:04:35 2009
@@ -18,11 +18,10 @@
  */
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
 
-public class TestISOLatin1AccentFilter extends LuceneTestCase {
+public class TestISOLatin1AccentFilter extends BaseTokenStreamTestCase {
   public void testU() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"));
     ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java Sat Aug 22 12:04:35 2009
@@ -31,9 +31,8 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
 
-public class TestKeywordAnalyzer extends LuceneTestCase {
+public class TestKeywordAnalyzer extends BaseTokenStreamTestCase {
   
   private RAMDirectory directory;
   private IndexSearcher searcher;

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java Sat Aug 22 12:04:35 2009
@@ -18,11 +18,10 @@
  */
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
 
-public class TestLengthFilter extends LuceneTestCase {
+public class TestLengthFilter extends BaseTokenStreamTestCase {
   
   public void testFilter() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Sat Aug 22 12:04:35 2009
@@ -17,12 +17,11 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
-public class TestNumericTokenStream extends LuceneTestCase {
+public class TestNumericTokenStream extends BaseTokenStreamTestCase {
 
   static final long lvalue = 4573245871874382L;
   static final int ivalue = 123456;

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Sat Aug 22 12:04:35 2009
@@ -3,7 +3,6 @@
 import java.io.StringReader;
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,7 +21,7 @@
  * limitations under the License.
  */
 
-public class TestPerFieldAnalzyerWrapper extends LuceneTestCase {
+public class TestPerFieldAnalzyerWrapper extends BaseTokenStreamTestCase {
   public void testPerField() throws Exception {
     String text = "Qwerty";
     PerFieldAnalyzerWrapper analyzer =

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Sat Aug 22 12:04:35 2009
@@ -5,7 +5,6 @@
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
 
@@ -25,7 +24,7 @@
  * limitations under the License.
  */
 
-public class TestStandardAnalyzer extends LuceneTestCase {
+public class TestStandardAnalyzer extends BaseTokenStreamTestCase {
 
   private Analyzer a = new StandardAnalyzer();
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Sat Aug 22 12:04:35 2009
@@ -19,7 +19,6 @@
 
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
 import java.io.IOException;
@@ -27,7 +26,7 @@
 import java.util.Set;
 import java.util.HashSet;
 
-public class TestStopAnalyzer extends LuceneTestCase {
+public class TestStopAnalyzer extends BaseTokenStreamTestCase {
   
   private StopAnalyzer stop = new StopAnalyzer(false);
   private Set inValidTokens = new HashSet();

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java Sat Aug 22 12:04:35 2009
@@ -19,7 +19,6 @@
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.English;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -27,7 +26,7 @@
 import java.util.Set;
 
 
-public class TestStopFilter extends LuceneTestCase {
+public class TestStopFilter extends BaseTokenStreamTestCase {
 
   private final static boolean VERBOSE = false;
   

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java Sat Aug 22 12:04:35 2009
@@ -22,7 +22,6 @@
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.English;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -32,7 +31,7 @@
 /**
  * tests for the TestTeeSinkTokenFilter
  */
-public class TestTeeSinkTokenFilter extends LuceneTestCase {
+public class TestTeeSinkTokenFilter extends BaseTokenStreamTestCase {
   protected StringBuffer buffer1;
   protected StringBuffer buffer2;
   protected String[] tokens1;

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Aug 22 12:04:35 2009
@@ -38,10 +38,10 @@
 import org.apache.lucene.document.Field.TermVector;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.util._TestUtil;
 
-public class TestDocumentWriter extends LuceneTestCase {
+public class TestDocumentWriter extends BaseTokenStreamTestCase {
   private RAMDirectory dir;
 
   public TestDocumentWriter(String s) {

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Sat Aug 22 12:04:35 2009
@@ -30,7 +30,7 @@
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 
 /**
  * Test QueryParser's ability to deal with Analyzers that return more
@@ -38,7 +38,7 @@
  * increment &gt; 1.
  *
  */
-public class TestMultiAnalyzer extends LuceneTestCase {
+public class TestMultiAnalyzer extends BaseTokenStreamTestCase {
 
   private static int multiToken = 0;
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java Sat Aug 22 12:04:35 2009
@@ -35,12 +35,12 @@
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 
 /**
  * Tests QueryParser.
  */
-public class TestMultiFieldQueryParser extends LuceneTestCase {
+public class TestMultiFieldQueryParser extends BaseTokenStreamTestCase {
 
   /** test stop words arsing for both the non static form, and for the 
    * corresponding static form (qtxt, fields[]). */

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java Sat Aug 22 12:04:35 2009
@@ -58,12 +58,12 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 
 /**
  * Tests QueryParser.
  */
-public class TestQueryParser extends LuceneTestCase {
+public class TestQueryParser extends BaseTokenStreamTestCase {
 
   public static Analyzer qpAnalyzer = new QPTestAnalyzer();
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java Sat Aug 22 12:04:35 2009
@@ -40,7 +40,7 @@
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.LowerCaseTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.index.Payload;
@@ -56,7 +56,7 @@
  *
  * @version $Revision$
  */
-public class TestPositionIncrement extends LuceneTestCase {
+public class TestPositionIncrement extends BaseTokenStreamTestCase {
 
   public void testSetPosition() throws Exception {
     Analyzer analyzer = new Analyzer() {

Modified: lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=806844&r1=806843&r2=806844&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/LuceneTestCase.java Sat Aug 22 12:04:35 2009
@@ -24,6 +24,7 @@
 
 import junit.framework.TestCase;
 
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.FieldCache.CacheEntry;
@@ -58,7 +59,9 @@
   }
 
   protected void setUp() throws Exception {
+    super.setUp();
     ConcurrentMergeScheduler.setTestMode();
+    TokenStream.setOnlyUseNewAPI(false);
   }
 
   /**
@@ -96,6 +99,7 @@
     } finally {
       purgeFieldCache(FieldCache.DEFAULT);
     }
+    super.tearDown();
   }
 
   /**