You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/10/07 08:24:27 UTC
svn commit: r1529770 [1/2] - in /lucene/dev/trunk:
lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/
lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellan...
Author: rmuir
Date: Mon Oct 7 06:24:25 2013
New Revision: 1529770
URL: http://svn.apache.org/r1529770
Log:
LUCENE-5259: convert analysis consumers to try-with-resources
Modified:
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
lucene/dev/trunk/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java
lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java
lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Mon Oct 7 06:24:25 2013
@@ -307,30 +307,30 @@ public class SynonymMap {
* separates by {@link SynonymMap#WORD_SEPARATOR}.
* reuse and its chars must not be null. */
public CharsRef analyze(String text, CharsRef reuse) throws IOException {
- TokenStream ts = analyzer.tokenStream("", text);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
- ts.reset();
- reuse.length = 0;
- while (ts.incrementToken()) {
- int length = termAtt.length();
- if (length == 0) {
- throw new IllegalArgumentException("term: " + text + " analyzed to a zero-length token");
+ try (TokenStream ts = analyzer.tokenStream("", text)) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
+ ts.reset();
+ reuse.length = 0;
+ while (ts.incrementToken()) {
+ int length = termAtt.length();
+ if (length == 0) {
+ throw new IllegalArgumentException("term: " + text + " analyzed to a zero-length token");
+ }
+ if (posIncAtt.getPositionIncrement() != 1) {
+ throw new IllegalArgumentException("term: " + text + " analyzed to a token with posinc != 1");
+ }
+ reuse.grow(reuse.length + length + 1); /* current + word + separator */
+ int end = reuse.offset + reuse.length;
+ if (reuse.length > 0) {
+ reuse.chars[end++] = SynonymMap.WORD_SEPARATOR;
+ reuse.length++;
+ }
+ System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
+ reuse.length += length;
}
- if (posIncAtt.getPositionIncrement() != 1) {
- throw new IllegalArgumentException("term: " + text + " analyzed to a token with posinc != 1");
- }
- reuse.grow(reuse.length + length + 1); /* current + word + separator */
- int end = reuse.offset + reuse.length;
- if (reuse.length > 0) {
- reuse.chars[end++] = SynonymMap.WORD_SEPARATOR;
- reuse.length++;
- }
- System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
- reuse.length += length;
+ ts.end();
}
- ts.end();
- ts.close();
if (reuse.length == 0) {
throw new IllegalArgumentException("term: " + text + " was completely eliminated by analyzer");
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java Mon Oct 7 06:24:25 2013
@@ -117,12 +117,15 @@ public class TestKeywordAnalyzer extends
// LUCENE-1441
public void testOffsets() throws Exception {
- TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"));
- OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
- stream.reset();
- assertTrue(stream.incrementToken());
- assertEquals(0, offsetAtt.startOffset());
- assertEquals(4, offsetAtt.endOffset());
+ try (TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"))) {
+ OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
+ stream.reset();
+ assertTrue(stream.incrementToken());
+ assertEquals(0, offsetAtt.startOffset());
+ assertEquals(4, offsetAtt.endOffset());
+ assertFalse(stream.incrementToken());
+ stream.end();
+ }
}
/** blast some random strings through the analyzer */
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java Mon Oct 7 06:24:25 2013
@@ -46,27 +46,31 @@ public class TestStopAnalyzer extends Ba
public void testDefaults() throws IOException {
assertTrue(stop != null);
- TokenStream stream = stop.tokenStream("test", "This is a test of the english stop analyzer");
- assertTrue(stream != null);
- CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
- stream.reset();
+ try (TokenStream stream = stop.tokenStream("test", "This is a test of the english stop analyzer")) {
+ assertTrue(stream != null);
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ stream.reset();
- while (stream.incrementToken()) {
- assertFalse(inValidTokens.contains(termAtt.toString()));
+ while (stream.incrementToken()) {
+ assertFalse(inValidTokens.contains(termAtt.toString()));
+ }
+ stream.end();
}
}
public void testStopList() throws IOException {
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("good", "test", "analyzer"), false);
StopAnalyzer newStop = new StopAnalyzer(Version.LUCENE_40, stopWordsSet);
- TokenStream stream = newStop.tokenStream("test", "This is a good test of the english stop analyzer");
- assertNotNull(stream);
- CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ try (TokenStream stream = newStop.tokenStream("test", "This is a good test of the english stop analyzer")) {
+ assertNotNull(stream);
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
- stream.reset();
- while (stream.incrementToken()) {
- String text = termAtt.toString();
- assertFalse(stopWordsSet.contains(text));
+ stream.reset();
+ while (stream.incrementToken()) {
+ String text = termAtt.toString();
+ assertFalse(stopWordsSet.contains(text));
+ }
+ stream.end();
}
}
@@ -75,17 +79,19 @@ public class TestStopAnalyzer extends Ba
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
String s = "This is a good test of the english stop analyzer with positions";
int expectedIncr[] = { 1, 1, 1, 3, 1, 1, 1, 2, 1};
- TokenStream stream = newStop.tokenStream("test", s);
- assertNotNull(stream);
- int i = 0;
- CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
- PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
+ try (TokenStream stream = newStop.tokenStream("test", s)) {
+ assertNotNull(stream);
+ int i = 0;
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
- stream.reset();
- while (stream.incrementToken()) {
- String text = termAtt.toString();
- assertFalse(stopWordsSet.contains(text));
- assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
+ stream.reset();
+ while (stream.incrementToken()) {
+ String text = termAtt.toString();
+ assertFalse(stopWordsSet.contains(text));
+ assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
+ }
+ stream.end();
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Mon Oct 7 06:24:25 2013
@@ -37,23 +37,29 @@ public class TestPerFieldAnalyzerWrapper
PerFieldAnalyzerWrapper analyzer =
new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
- TokenStream tokenStream = analyzer.tokenStream("field", text);
- CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
- tokenStream.reset();
+ try (TokenStream tokenStream = analyzer.tokenStream("field", text)) {
+ CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
+ tokenStream.reset();
- assertTrue(tokenStream.incrementToken());
- assertEquals("WhitespaceAnalyzer does not lowercase",
+ assertTrue(tokenStream.incrementToken());
+ assertEquals("WhitespaceAnalyzer does not lowercase",
"Qwerty",
termAtt.toString());
+ assertFalse(tokenStream.incrementToken());
+ tokenStream.end();
+ }
- tokenStream = analyzer.tokenStream("special", text);
- termAtt = tokenStream.getAttribute(CharTermAttribute.class);
- tokenStream.reset();
+ try (TokenStream tokenStream = analyzer.tokenStream("special", text)) {
+ CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
+ tokenStream.reset();
- assertTrue(tokenStream.incrementToken());
- assertEquals("SimpleAnalyzer lowercases",
+ assertTrue(tokenStream.incrementToken());
+ assertEquals("SimpleAnalyzer lowercases",
"qwerty",
termAtt.toString());
+ assertFalse(tokenStream.incrementToken());
+ tokenStream.end();
+ }
}
public void testCharFilters() throws Exception {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Mon Oct 7 06:24:25 2013
@@ -95,17 +95,19 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperPhraseQuery() throws Exception {
PhraseQuery q = new PhraseQuery();
- TokenStream ts = analyzer.tokenStream("content", "this sentence");
- int j = -1;
+ try (TokenStream ts = analyzer.tokenStream("content", "this sentence")) {
+ int j = -1;
- PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- j += posIncrAtt.getPositionIncrement();
- String termText = termAtt.toString();
- q.add(new Term("content", termText), j);
+ ts.reset();
+ while (ts.incrementToken()) {
+ j += posIncrAtt.getPositionIncrement();
+ String termText = termAtt.toString();
+ q.add(new Term("content", termText), j);
+ }
+ ts.end();
}
ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;
@@ -121,16 +123,16 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperBooleanQuery() throws Exception {
BooleanQuery q = new BooleanQuery();
- TokenStream ts = analyzer.tokenStream("content", "test sentence");
+ try (TokenStream ts = analyzer.tokenStream("content", "test sentence")) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
- ts.reset();
-
- while (ts.incrementToken()) {
- String termText = termAtt.toString();
- q.add(new TermQuery(new Term("content", termText)),
+ ts.reset();
+ while (ts.incrementToken()) {
+ String termText = termAtt.toString();
+ q.add(new TermQuery(new Term("content", termText)),
BooleanClause.Occur.SHOULD);
+ }
+ ts.end();
}
ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Mon Oct 7 06:24:25 2013
@@ -123,18 +123,18 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", s);
- ts.reset();
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- while (ts.incrementToken()) {
- String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
- for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
- cp = highlightedText.codePointAt(j);
- assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ try (TokenStream ts = analyzer.tokenStream("foo", s)) {
+ ts.reset();
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+ while (ts.incrementToken()) {
+ String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
+ for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
+ cp = highlightedText.codePointAt(j);
+ assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ }
}
+ ts.end();
}
- ts.end();
- ts.close();
}
// just for fun
checkRandomData(random(), analyzer, num);
@@ -161,18 +161,18 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", s);
- ts.reset();
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- while (ts.incrementToken()) {
- String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
- for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
- cp = highlightedText.codePointAt(j);
- assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ try (TokenStream ts = analyzer.tokenStream("foo", s)) {
+ ts.reset();
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+ while (ts.incrementToken()) {
+ String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
+ for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
+ cp = highlightedText.codePointAt(j);
+ assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ }
}
+ ts.end();
}
- ts.end();
- ts.close();
}
// just for fun
checkRandomData(random(), analyzer, num);
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Mon Oct 7 06:24:25 2013
@@ -249,16 +249,16 @@ public class TestICUTokenizer extends Ba
}
public void testTokenAttributes() throws Exception {
- TokenStream ts = a.tokenStream("dummy", "This is a test");
- ScriptAttribute scriptAtt = ts.addAttribute(ScriptAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- assertEquals(UScript.LATIN, scriptAtt.getCode());
- assertEquals(UScript.getName(UScript.LATIN), scriptAtt.getName());
- assertEquals(UScript.getShortName(UScript.LATIN), scriptAtt.getShortName());
- assertTrue(ts.reflectAsString(false).contains("script=Latin"));
+ try (TokenStream ts = a.tokenStream("dummy", "This is a test")) {
+ ScriptAttribute scriptAtt = ts.addAttribute(ScriptAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ assertEquals(UScript.LATIN, scriptAtt.getCode());
+ assertEquals(UScript.getName(UScript.LATIN), scriptAtt.getName());
+ assertEquals(UScript.getShortName(UScript.LATIN), scriptAtt.getShortName());
+ assertTrue(ts.reflectAsString(false).contains("script=Latin"));
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Mon Oct 7 06:24:25 2013
@@ -53,14 +53,14 @@ public class TestExtendedMode extends Ba
int numIterations = atLeast(1000);
for (int i = 0; i < numIterations; i++) {
String s = _TestUtil.randomUnicodeString(random(), 100);
- TokenStream ts = analyzer.tokenStream("foo", s);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- assertTrue(UnicodeUtil.validUTF16String(termAtt));
+ try (TokenStream ts = analyzer.tokenStream("foo", s)) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ assertTrue(UnicodeUtil.validUTF16String(termAtt));
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java Mon Oct 7 06:24:25 2013
@@ -141,13 +141,13 @@ public class TestJapaneseTokenizer exten
* ideally the test would actually fail instead of hanging...
*/
public void testDecomposition5() throws Exception {
- TokenStream ts = analyzer.tokenStream("bogus", "ãããããããããããããããããããããããããããããããããããããããã");
- ts.reset();
- while (ts.incrementToken()) {
+ try (TokenStream ts = analyzer.tokenStream("bogus", "ãããããããããããããããããããããããããããããããããããããããã")) {
+ ts.reset();
+ while (ts.incrementToken()) {
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
/*
@@ -213,12 +213,12 @@ public class TestJapaneseTokenizer exten
public void testLargeDocReliability() throws Exception {
for (int i = 0; i < 100; i++) {
String s = _TestUtil.randomUnicodeString(random(), 10000);
- TokenStream ts = analyzer.tokenStream("foo", s);
- ts.reset();
- while (ts.incrementToken()) {
+ try (TokenStream ts = analyzer.tokenStream("foo", s)) {
+ ts.reset();
+ while (ts.incrementToken()) {
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
}
@@ -236,29 +236,31 @@ public class TestJapaneseTokenizer exten
System.out.println("\nTEST: iter=" + i);
}
String s = _TestUtil.randomUnicodeString(random(), 100);
- TokenStream ts = analyzer.tokenStream("foo", s);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- assertTrue(UnicodeUtil.validUTF16String(termAtt));
+ try (TokenStream ts = analyzer.tokenStream("foo", s)) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ assertTrue(UnicodeUtil.validUTF16String(termAtt));
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
}
public void testOnlyPunctuation() throws IOException {
- TokenStream ts = analyzerNoPunct.tokenStream("foo", "ãããã");
- ts.reset();
- assertFalse(ts.incrementToken());
- ts.end();
+ try (TokenStream ts = analyzerNoPunct.tokenStream("foo", "ãããã")) {
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
}
public void testOnlyPunctuationExtended() throws IOException {
- TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", "......");
- ts.reset();
- assertFalse(ts.incrementToken());
- ts.end();
+ try (TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", "......")) {
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
}
// note: test is kinda silly since kuromoji emits punctuation tokens.
@@ -369,75 +371,81 @@ public class TestJapaneseTokenizer exten
}
private void assertReadings(String input, String... readings) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
- ts.reset();
- for(String reading : readings) {
- assertTrue(ts.incrementToken());
- assertEquals(reading, readingAtt.getReading());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
+ ts.reset();
+ for(String reading : readings) {
+ assertTrue(ts.incrementToken());
+ assertEquals(reading, readingAtt.getReading());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
private void assertPronunciations(String input, String... pronunciations) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
- ts.reset();
- for(String pronunciation : pronunciations) {
- assertTrue(ts.incrementToken());
- assertEquals(pronunciation, readingAtt.getPronunciation());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
+ ts.reset();
+ for(String pronunciation : pronunciations) {
+ assertTrue(ts.incrementToken());
+ assertEquals(pronunciation, readingAtt.getPronunciation());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
private void assertBaseForms(String input, String... baseForms) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- BaseFormAttribute baseFormAtt = ts.addAttribute(BaseFormAttribute.class);
- ts.reset();
- for(String baseForm : baseForms) {
- assertTrue(ts.incrementToken());
- assertEquals(baseForm, baseFormAtt.getBaseForm());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ BaseFormAttribute baseFormAtt = ts.addAttribute(BaseFormAttribute.class);
+ ts.reset();
+ for(String baseForm : baseForms) {
+ assertTrue(ts.incrementToken());
+ assertEquals(baseForm, baseFormAtt.getBaseForm());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
private void assertInflectionTypes(String input, String... inflectionTypes) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
- ts.reset();
- for(String inflectionType : inflectionTypes) {
- assertTrue(ts.incrementToken());
- assertEquals(inflectionType, inflectionAtt.getInflectionType());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
+ ts.reset();
+ for(String inflectionType : inflectionTypes) {
+ assertTrue(ts.incrementToken());
+ assertEquals(inflectionType, inflectionAtt.getInflectionType());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
private void assertInflectionForms(String input, String... inflectionForms) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
- ts.reset();
- for(String inflectionForm : inflectionForms) {
- assertTrue(ts.incrementToken());
- assertEquals(inflectionForm, inflectionAtt.getInflectionForm());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
+ ts.reset();
+ for(String inflectionForm : inflectionForms) {
+ assertTrue(ts.incrementToken());
+ assertEquals(inflectionForm, inflectionAtt.getInflectionForm());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
private void assertPartsOfSpeech(String input, String... partsOfSpeech) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", input);
- PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);
- ts.reset();
- for(String partOfSpeech : partsOfSpeech) {
- assertTrue(ts.incrementToken());
- assertEquals(partOfSpeech, partOfSpeechAtt.getPartOfSpeech());
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);
+ ts.reset();
+ for(String partOfSpeech : partsOfSpeech) {
+ assertTrue(ts.incrementToken());
+ assertEquals(partOfSpeech, partOfSpeechAtt.getPartOfSpeech());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
}
- assertFalse(ts.incrementToken());
- ts.end();
}
public void testReadings() throws Exception {
@@ -631,11 +639,11 @@ public class TestJapaneseTokenizer exten
long totalStart = System.currentTimeMillis();
for (int i = 0; i < numIterations; i++) {
- final TokenStream ts = analyzer.tokenStream("ignored", line);
- ts.reset();
- while(ts.incrementToken());
- ts.end();
- ts.close();
+ try (TokenStream ts = analyzer.tokenStream("ignored", line)) {
+ ts.reset();
+ while(ts.incrementToken());
+ ts.end();
+ }
}
String[] sentences = line.split("ã|ã");
if (VERBOSE) {
@@ -645,11 +653,11 @@ public class TestJapaneseTokenizer exten
totalStart = System.currentTimeMillis();
for (int i = 0; i < numIterations; i++) {
for (String sentence: sentences) {
- final TokenStream ts = analyzer.tokenStream("ignored", sentence);
- ts.reset();
- while(ts.incrementToken());
- ts.end();
- ts.close();
+ try (TokenStream ts = analyzer.tokenStream("ignored", sentence)) {
+ ts.reset();
+ while(ts.incrementToken());
+ ts.end();
+ }
}
}
if (VERBOSE) {
Modified: lucene/dev/trunk/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java Mon Oct 7 06:24:25 2013
@@ -72,34 +72,36 @@ public class TestMorfologikAnalyzer exte
@SuppressWarnings("unused")
private void dumpTokens(String input) throws IOException {
- TokenStream ts = getTestAnalyzer().tokenStream("dummy", input);
- ts.reset();
+ try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", input)) {
+ ts.reset();
- MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class);
- CharTermAttribute charTerm = ts.getAttribute(CharTermAttribute.class);
- while (ts.incrementToken()) {
- System.out.println(charTerm.toString() + " => " + attribute.getTags());
+ MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class);
+ CharTermAttribute charTerm = ts.getAttribute(CharTermAttribute.class);
+ while (ts.incrementToken()) {
+ System.out.println(charTerm.toString() + " => " + attribute.getTags());
+ }
+ ts.end();
}
}
/** Test reuse of MorfologikFilter with leftover stems. */
public final void testLeftoverStems() throws IOException {
Analyzer a = getTestAnalyzer();
- TokenStream ts_1 = a.tokenStream("dummy", "liÅcie");
- CharTermAttribute termAtt_1 = ts_1.getAttribute(CharTermAttribute.class);
- ts_1.reset();
- ts_1.incrementToken();
- assertEquals("first stream", "liÅcie", termAtt_1.toString());
- ts_1.end();
- ts_1.close();
-
- TokenStream ts_2 = a.tokenStream("dummy", "danych");
- CharTermAttribute termAtt_2 = ts_2.getAttribute(CharTermAttribute.class);
- ts_2.reset();
- ts_2.incrementToken();
- assertEquals("second stream", "dany", termAtt_2.toString());
- ts_2.end();
- ts_2.close();
+ try (TokenStream ts_1 = a.tokenStream("dummy", "liÅcie")) {
+ CharTermAttribute termAtt_1 = ts_1.getAttribute(CharTermAttribute.class);
+ ts_1.reset();
+ ts_1.incrementToken();
+ assertEquals("first stream", "liÅcie", termAtt_1.toString());
+ ts_1.end();
+ }
+
+ try (TokenStream ts_2 = a.tokenStream("dummy", "danych")) {
+ CharTermAttribute termAtt_2 = ts_2.getAttribute(CharTermAttribute.class);
+ ts_2.reset();
+ ts_2.incrementToken();
+ assertEquals("second stream", "dany", termAtt_2.toString());
+ ts_2.end();
+ }
}
/** Test stemming of mixed-case tokens. */
@@ -140,28 +142,27 @@ public class TestMorfologikAnalyzer exte
/** Test morphosyntactic annotations. */
public final void testPOSAttribute() throws IOException {
- TokenStream ts = getTestAnalyzer().tokenStream("dummy", "liÅcie");
-
- ts.reset();
- assertPOSToken(ts, "liÅcie",
+ try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", "liÅcie")) {
+ ts.reset();
+ assertPOSToken(ts, "liÅcie",
"subst:sg:acc:n2",
"subst:sg:nom:n2",
"subst:sg:voc:n2");
- assertPOSToken(ts, "liÅÄ",
+ assertPOSToken(ts, "liÅÄ",
"subst:pl:acc:m3",
"subst:pl:nom:m3",
"subst:pl:voc:m3");
- assertPOSToken(ts, "list",
+ assertPOSToken(ts, "list",
"subst:sg:loc:m3",
"subst:sg:voc:m3");
- assertPOSToken(ts, "lista",
+ assertPOSToken(ts, "lista",
"subst:sg:dat:f",
"subst:sg:loc:f");
- ts.end();
- ts.close();
+ ts.end();
+ }
}
/** */
Modified: lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java Mon Oct 7 06:24:25 2013
@@ -184,9 +184,11 @@ public class TestSmartChineseAnalyzer ex
sb.append("æè´ä¹°äºéå
·åæè£
ã");
}
Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT);
- TokenStream stream = analyzer.tokenStream("", sb.toString());
- stream.reset();
- while (stream.incrementToken()) {
+ try (TokenStream stream = analyzer.tokenStream("", sb.toString())) {
+ stream.reset();
+ while (stream.incrementToken()) {
+ }
+ stream.end();
}
}
@@ -197,9 +199,11 @@ public class TestSmartChineseAnalyzer ex
sb.append("æè´ä¹°äºéå
·åæè£
");
}
Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT);
- TokenStream stream = analyzer.tokenStream("", sb.toString());
- stream.reset();
- while (stream.incrementToken()) {
+ try (TokenStream stream = analyzer.tokenStream("", sb.toString())) {
+ stream.reset();
+ while (stream.incrementToken()) {
+ }
+ stream.end();
}
}
Modified: lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java (original)
+++ lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java Mon Oct 7 06:24:25 2013
@@ -91,20 +91,19 @@ public class BooleanPerceptronClassifier
throw new IOException("You must first call Classifier#train");
}
Long output = 0l;
- TokenStream tokenStream = analyzer.tokenStream(textFieldName,
- new StringReader(text));
- CharTermAttribute charTermAttribute = tokenStream
+ try (TokenStream tokenStream = analyzer.tokenStream(textFieldName, text)) {
+ CharTermAttribute charTermAttribute = tokenStream
.addAttribute(CharTermAttribute.class);
- tokenStream.reset();
- while (tokenStream.incrementToken()) {
- String s = charTermAttribute.toString();
- Long d = Util.get(fst, new BytesRef(s));
- if (d != null) {
- output += d;
+ tokenStream.reset();
+ while (tokenStream.incrementToken()) {
+ String s = charTermAttribute.toString();
+ Long d = Util.get(fst, new BytesRef(s));
+ if (d != null) {
+ output += d;
+ }
}
+ tokenStream.end();
}
- tokenStream.end();
- tokenStream.close();
return new ClassificationResult<>(output >= threshold, output.doubleValue());
}
Modified: lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java (original)
+++ lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java Mon Oct 7 06:24:25 2013
@@ -85,14 +85,14 @@ public class SimpleNaiveBayesClassifier
private String[] tokenizeDoc(String doc) throws IOException {
Collection<String> result = new LinkedList<String>();
- TokenStream tokenStream = analyzer.tokenStream(textFieldName, doc);
- CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
- tokenStream.reset();
- while (tokenStream.incrementToken()) {
- result.add(charTermAttribute.toString());
+ try (TokenStream tokenStream = analyzer.tokenStream(textFieldName, doc)) {
+ CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
+ tokenStream.reset();
+ while (tokenStream.incrementToken()) {
+ result.add(charTermAttribute.toString());
+ }
+ tokenStream.end();
}
- tokenStream.end();
- tokenStream.close();
return result.toArray(new String[result.size()]);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Mon Oct 7 06:24:25 2013
@@ -92,13 +92,9 @@ final class DocInverterPerField extends
fieldState.position += analyzed ? docState.analyzer.getPositionIncrementGap(fieldInfo.name) : 0;
}
- final TokenStream stream = field.tokenStream(docState.analyzer);
- // reset the TokenStream to the first token
- stream.reset();
-
- boolean success2 = false;
-
- try {
+ try (TokenStream stream = field.tokenStream(docState.analyzer)) {
+ // reset the TokenStream to the first token
+ stream.reset();
boolean hasMoreTokens = stream.incrementToken();
fieldState.attributeSource = stream;
@@ -179,13 +175,6 @@ final class DocInverterPerField extends
// when we come back around to the field...
fieldState.position += posIncrAttribute.getPositionIncrement();
fieldState.offset += offsetAttribute.endOffset();
- success2 = true;
- } finally {
- if (!success2) {
- IOUtils.closeWhileHandlingException(stream);
- } else {
- stream.close();
- }
}
fieldState.offset += analyzed ? docState.analyzer.getOffsetGap(fieldInfo.name) : 0;
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Mon Oct 7 06:24:25 2013
@@ -98,13 +98,13 @@ public class TestMockAnalyzer extends Ba
String testString = "t";
Analyzer analyzer = new MockAnalyzer(random());
- TokenStream stream = analyzer.tokenStream("dummy", testString);
- stream.reset();
- while (stream.incrementToken()) {
- // consume
+ try (TokenStream stream = analyzer.tokenStream("dummy", testString)) {
+ stream.reset();
+ while (stream.incrementToken()) {
+ // consume
+ }
+ stream.end();
}
- stream.end();
- stream.close();
assertAnalyzesTo(analyzer, testString, new String[] { "t" });
}
@@ -121,13 +121,13 @@ public class TestMockAnalyzer extends Ba
StringReader reader = new StringReader(s);
MockCharFilter charfilter = new MockCharFilter(reader, 2);
MockAnalyzer analyzer = new MockAnalyzer(random());
- TokenStream ts = analyzer.tokenStream("bogus", charfilter);
- ts.reset();
- while (ts.incrementToken()) {
- ;
+ try (TokenStream ts = analyzer.tokenStream("bogus", charfilter)) {
+ ts.reset();
+ while (ts.incrementToken()) {
+ ;
+ }
+ ts.end();
}
- ts.end();
- ts.close();
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Mon Oct 7 06:24:25 2013
@@ -47,30 +47,29 @@ public class TestLongPostings extends Lu
if (other != null && s.equals(other)) {
continue;
}
- final TokenStream ts = a.tokenStream("foo", s);
- final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
- final BytesRef termBytes = termAtt.getBytesRef();
- ts.reset();
-
- int count = 0;
- boolean changed = false;
-
- while(ts.incrementToken()) {
- termAtt.fillBytesRef();
- if (count == 0 && !termBytes.utf8ToString().equals(s)) {
- // The value was changed during analysis. Keep iterating so the
- // tokenStream is exhausted.
- changed = true;
+ try (TokenStream ts = a.tokenStream("foo", s)) {
+ final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
+ final BytesRef termBytes = termAtt.getBytesRef();
+ ts.reset();
+
+ int count = 0;
+ boolean changed = false;
+
+ while(ts.incrementToken()) {
+ termAtt.fillBytesRef();
+ if (count == 0 && !termBytes.utf8ToString().equals(s)) {
+ // The value was changed during analysis. Keep iterating so the
+ // tokenStream is exhausted.
+ changed = true;
+ }
+ count++;
}
- count++;
- }
- ts.end();
- ts.close();
-
- // Did we iterate just once and the value was unchanged?
- if (!changed && count == 1) {
- return s;
+ ts.end();
+ // Did we iterate just once and the value was unchanged?
+ if (!changed && count == 1) {
+ return s;
+ }
}
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Mon Oct 7 06:24:25 2013
@@ -174,17 +174,18 @@ public class TestTermVectorsWriter exten
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream stream = analyzer.tokenStream("field", "abcd ");
- stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
- stream = new CachingTokenFilter(stream);
- FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
- Field f = new Field("field", stream, customType);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
+ try (TokenStream stream = analyzer.tokenStream("field", "abcd ")) {
+ stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
+ TokenStream cachedStream = new CachingTokenFilter(stream);
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectorOffsets(true);
+ Field f = new Field("field", cachedStream, customType);
+ doc.add(f);
+ doc.add(f);
+ w.addDocument(doc);
+ }
w.close();
IndexReader r = DirectoryReader.open(dir);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java Mon Oct 7 06:24:25 2013
@@ -617,16 +617,16 @@ public class TestPhraseQuery extends Luc
break;
}
}
- TokenStream ts = analyzer.tokenStream("ignore", term);
- CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while(ts.incrementToken()) {
- String text = termAttr.toString();
- doc.add(text);
- sb.append(text).append(' ');
+ try (TokenStream ts = analyzer.tokenStream("ignore", term)) {
+ CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while(ts.incrementToken()) {
+ String text = termAttr.toString();
+ doc.add(text);
+ sb.append(text).append(' ');
+ }
+ ts.end();
}
- ts.end();
- ts.close();
} else {
// pick existing sub-phrase
List<String> lastDoc = docs.get(r.nextInt(docs.size()));
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Mon Oct 7 06:24:25 2013
@@ -170,20 +170,20 @@ public abstract class AbstractTestCase e
protected List<BytesRef> analyze(String text, String field, Analyzer analyzer) throws IOException {
List<BytesRef> bytesRefs = new ArrayList<BytesRef>();
- TokenStream tokenStream = analyzer.tokenStream(field, text);
- TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);
+ try (TokenStream tokenStream = analyzer.tokenStream(field, text)) {
+ TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytesRef = termAttribute.getBytesRef();
+ BytesRef bytesRef = termAttribute.getBytesRef();
- tokenStream.reset();
+ tokenStream.reset();
- while (tokenStream.incrementToken()) {
- termAttribute.fillBytesRef();
- bytesRefs.add(BytesRef.deepCopyOf(bytesRef));
- }
+ while (tokenStream.incrementToken()) {
+ termAttribute.fillBytesRef();
+ bytesRefs.add(BytesRef.deepCopyOf(bytesRef));
+ }
- tokenStream.end();
- tokenStream.close();
+ tokenStream.end();
+ }
return bytesRefs;
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java Mon Oct 7 06:24:25 2013
@@ -777,31 +777,31 @@ public final class MoreLikeThis {
throw new UnsupportedOperationException("To use MoreLikeThis without " +
"term vectors, you must provide an Analyzer");
}
- TokenStream ts = analyzer.tokenStream(fieldName, r);
- int tokenCount = 0;
- // for every token
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- String word = termAtt.toString();
- tokenCount++;
- if (tokenCount > maxNumTokensParsed) {
- break;
- }
- if (isNoiseWord(word)) {
- continue;
- }
-
- // increment frequency
- Int cnt = termFreqMap.get(word);
- if (cnt == null) {
- termFreqMap.put(word, new Int());
- } else {
- cnt.x++;
+ try (TokenStream ts = analyzer.tokenStream(fieldName, r)) {
+ int tokenCount = 0;
+ // for every token
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ String word = termAtt.toString();
+ tokenCount++;
+ if (tokenCount > maxNumTokensParsed) {
+ break;
+ }
+ if (isNoiseWord(word)) {
+ continue;
+ }
+
+ // increment frequency
+ Int cnt = termFreqMap.get(word);
+ if (cnt == null) {
+ termFreqMap.put(word, new Int());
+ } else {
+ cnt.x++;
+ }
}
+ ts.end();
}
- ts.end();
- ts.close();
}
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java Mon Oct 7 06:24:25 2013
@@ -162,9 +162,7 @@ public class AnalyzingQueryParser extend
*/
protected String analyzeSingleChunk(String field, String termStr, String chunk) throws ParseException{
String analyzed = null;
- TokenStream stream = null;
- try{
- stream = getAnalyzer().tokenStream(field, chunk);
+ try (TokenStream stream = getAnalyzer().tokenStream(field, chunk)) {
stream.reset();
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
// get first and hopefully only output token
@@ -186,7 +184,6 @@ public class AnalyzingQueryParser extend
multipleOutputs.append('"');
}
stream.end();
- stream.close();
if (null != multipleOutputs) {
throw new ParseException(
String.format(getLocale(),
@@ -196,7 +193,6 @@ public class AnalyzingQueryParser extend
// nothing returned by analyzer. Was it a stop word and the user accidentally
// used an analyzer with stop words?
stream.end();
- stream.close();
throw new ParseException(String.format(getLocale(), "Analyzer returned nothing for \"%s\"", chunk));
}
} catch (IOException e){
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Mon Oct 7 06:24:25 2013
@@ -497,63 +497,51 @@ public abstract class QueryParserBase im
protected Query newFieldQuery(Analyzer analyzer, String field, String queryText, boolean quoted) throws ParseException {
// Use the analyzer to get all the tokens, and then build a TermQuery,
// PhraseQuery, or nothing based on the term count
-
- TokenStream source;
- try {
- source = analyzer.tokenStream(field, queryText);
- source.reset();
- } catch (IOException e) {
- ParseException p = new ParseException("Unable to initialize TokenStream to analyze query text");
- p.initCause(e);
- throw p;
- }
- CachingTokenFilter buffer = new CachingTokenFilter(source);
+ CachingTokenFilter buffer = null;
TermToBytesRefAttribute termAtt = null;
PositionIncrementAttribute posIncrAtt = null;
int numTokens = 0;
-
- buffer.reset();
-
- if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
- termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
- }
- if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
- posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
- }
-
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
+ boolean hasMoreTokens = false;
+
+ try (TokenStream source = analyzer.tokenStream(field, queryText)) {
+ source.reset();
+ buffer = new CachingTokenFilter(source);
+ buffer.reset();
- boolean hasMoreTokens = false;
- if (termAtt != null) {
- try {
- hasMoreTokens = buffer.incrementToken();
- while (hasMoreTokens) {
- numTokens++;
- int positionIncrement = (posIncrAtt != null) ? posIncrAtt.getPositionIncrement() : 1;
- if (positionIncrement != 0) {
- positionCount += positionIncrement;
- } else {
- severalTokensAtSamePosition = true;
- }
+ if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
+ termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
+ }
+ if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
+ posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
+ }
+
+ if (termAtt != null) {
+ try {
hasMoreTokens = buffer.incrementToken();
+ while (hasMoreTokens) {
+ numTokens++;
+ int positionIncrement = (posIncrAtt != null) ? posIncrAtt.getPositionIncrement() : 1;
+ if (positionIncrement != 0) {
+ positionCount += positionIncrement;
+ } else {
+ severalTokensAtSamePosition = true;
+ }
+ hasMoreTokens = buffer.incrementToken();
+ }
+ } catch (IOException e) {
+ // ignore
}
- } catch (IOException e) {
- // ignore
}
- }
- try {
- // rewind the buffer stream
- buffer.reset();
-
- // close original stream - all tokens buffered
- source.close();
- }
- catch (IOException e) {
- ParseException p = new ParseException("Cannot close TokenStream analyzing query text");
+ } catch (IOException e) {
+ ParseException p = new ParseException("Eror analyzing query text");
p.initCause(e);
throw p;
}
+
+ // rewind the buffer stream
+ buffer.reset();
BytesRef bytes = termAtt == null ? null : termAtt.getBytesRef();
@@ -839,38 +827,24 @@ public abstract class QueryParserBase im
}
protected BytesRef analyzeMultitermTerm(String field, String part, Analyzer analyzerIn) {
- TokenStream source;
-
if (analyzerIn == null) analyzerIn = analyzer;
- try {
- source = analyzerIn.tokenStream(field, part);
+ try (TokenStream source = analyzerIn.tokenStream(field, part)) {
source.reset();
- } catch (IOException e) {
- throw new RuntimeException("Unable to initialize TokenStream to analyze multiTerm term: " + part, e);
- }
- TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
+ TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
+ BytesRef bytes = termAtt.getBytesRef();
- try {
if (!source.incrementToken())
throw new IllegalArgumentException("analyzer returned no terms for multiTerm term: " + part);
termAtt.fillBytesRef();
if (source.incrementToken())
throw new IllegalArgumentException("analyzer returned too many terms for multiTerm term: " + part);
- } catch (IOException e) {
- throw new RuntimeException("error analyzing range part: " + part, e);
- }
-
- try {
source.end();
- source.close();
+ return BytesRef.deepCopyOf(bytes);
} catch (IOException e) {
- throw new RuntimeException("Unable to end & close TokenStream after analyzing multiTerm term: " + part, e);
+ throw new RuntimeException("Error analyzing multiTerm term: " + part, e);
}
-
- return BytesRef.deepCopyOf(bytes);
}
/**
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java Mon Oct 7 06:24:25 2013
@@ -113,52 +113,44 @@ public class AnalyzerQueryNodeProcessor
String text = fieldNode.getTextAsString();
String field = fieldNode.getFieldAsString();
- TokenStream source;
- try {
- source = this.analyzer.tokenStream(field, text);
- source.reset();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
- }
- CachingTokenFilter buffer = new CachingTokenFilter(source);
-
+ CachingTokenFilter buffer = null;
PositionIncrementAttribute posIncrAtt = null;
int numTokens = 0;
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
+
+ try (TokenStream source = this.analyzer.tokenStream(field, text)) {
+ source.reset();
+ buffer = new CachingTokenFilter(source);
- if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
- posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
- }
+ if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
+ posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
+ }
- try {
+ try {
- while (buffer.incrementToken()) {
- numTokens++;
- int positionIncrement = (posIncrAtt != null) ? posIncrAtt
- .getPositionIncrement() : 1;
- if (positionIncrement != 0) {
- positionCount += positionIncrement;
+ while (buffer.incrementToken()) {
+ numTokens++;
+ int positionIncrement = (posIncrAtt != null) ? posIncrAtt
+ .getPositionIncrement() : 1;
+ if (positionIncrement != 0) {
+ positionCount += positionIncrement;
+
+ } else {
+ severalTokensAtSamePosition = true;
+ }
- } else {
- severalTokensAtSamePosition = true;
}
+ } catch (IOException e) {
+ // ignore
}
-
- } catch (IOException e) {
- // ignore
- }
-
- try {
- // rewind the buffer stream
- buffer.reset();
-
- // close original stream - all tokens buffered
- source.close();
} catch (IOException e) {
- // ignore
+ throw new RuntimeException(e);
}
+
+ // rewind the buffer stream
+ buffer.reset();
if (!buffer.hasAttribute(CharTermAttribute.class)) {
return new NoTokenFoundQueryNode();
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java Mon Oct 7 06:24:25 2013
@@ -73,8 +73,7 @@ public class LikeThisQueryBuilder implem
if ((stopWords != null) && (fields != null)) {
stopWordsSet = new HashSet<String>();
for (String field : fields) {
- try {
- TokenStream ts = analyzer.tokenStream(field, stopWords);
+ try (TokenStream ts = analyzer.tokenStream(field, stopWords)) {
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java Mon Oct 7 06:24:25 2013
@@ -49,9 +49,9 @@ public class SpanOrTermsBuilder extends
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
String value = DOMUtils.getNonBlankTextOrFail(e);
- try {
- List<SpanQuery> clausesList = new ArrayList<SpanQuery>();
- TokenStream ts = analyzer.tokenStream(fieldName, value);
+ List<SpanQuery> clausesList = new ArrayList<SpanQuery>();
+
+ try (TokenStream ts = analyzer.tokenStream(fieldName, value)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
@@ -61,7 +61,6 @@ public class SpanOrTermsBuilder extends
clausesList.add(stq);
}
ts.end();
- ts.close();
SpanOrQuery soq = new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
soq.setBoost(DOMUtils.getAttribute(e, "boost", 1.0f));
return soq;
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java Mon Oct 7 06:24:25 2013
@@ -54,8 +54,7 @@ public class TermsFilterBuilder implemen
String text = DOMUtils.getNonBlankTextOrFail(e);
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
- try {
- TokenStream ts = analyzer.tokenStream(fieldName, text);
+ try (TokenStream ts = analyzer.tokenStream(fieldName, text)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
@@ -64,7 +63,6 @@ public class TermsFilterBuilder implemen
terms.add(BytesRef.deepCopyOf(bytes));
}
ts.end();
- ts.close();
}
catch (IOException ioe) {
throw new RuntimeException("Error constructing terms from index:" + ioe);
Modified: lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java Mon Oct 7 06:24:25 2013
@@ -51,8 +51,7 @@ public class TermsQueryBuilder implement
BooleanQuery bq = new BooleanQuery(DOMUtils.getAttribute(e, "disableCoord", false));
bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e, "minimumNumberShouldMatch", 0));
- try {
- TokenStream ts = analyzer.tokenStream(fieldName, text);
+ try (TokenStream ts = analyzer.tokenStream(fieldName, text)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
@@ -63,7 +62,6 @@ public class TermsQueryBuilder implement
bq.add(new BooleanClause(new TermQuery(term), BooleanClause.Occur.SHOULD));
}
ts.end();
- ts.close();
}
catch (IOException ioe) {
throw new RuntimeException("Error constructing terms from index:" + ioe);
Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Mon Oct 7 06:24:25 2013
@@ -193,67 +193,67 @@ public class FuzzyLikeThisQuery extends
private void addTerms(IndexReader reader, FieldVals f) throws IOException {
if (f.queryString == null) return;
- TokenStream ts = analyzer.tokenStream(f.fieldName, f.queryString);
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
- int corpusNumDocs = reader.numDocs();
- HashSet<String> processedTerms = new HashSet<String>();
- ts.reset();
final Terms terms = MultiFields.getTerms(reader, f.fieldName);
if (terms == null) {
return;
}
- while (ts.incrementToken()) {
- String term = termAtt.toString();
- if (!processedTerms.contains(term)) {
- processedTerms.add(term);
- ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
- float minScore = 0;
- Term startTerm = new Term(f.fieldName, term);
- AttributeSource atts = new AttributeSource();
- MaxNonCompetitiveBoostAttribute maxBoostAtt =
+ try (TokenStream ts = analyzer.tokenStream(f.fieldName, f.queryString)) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+
+ int corpusNumDocs = reader.numDocs();
+ HashSet<String> processedTerms = new HashSet<String>();
+ ts.reset();
+ while (ts.incrementToken()) {
+ String term = termAtt.toString();
+ if (!processedTerms.contains(term)) {
+ processedTerms.add(term);
+ ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
+ float minScore = 0;
+ Term startTerm = new Term(f.fieldName, term);
+ AttributeSource atts = new AttributeSource();
+ MaxNonCompetitiveBoostAttribute maxBoostAtt =
atts.addAttribute(MaxNonCompetitiveBoostAttribute.class);
- SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
- //store the df so all variants use same idf
- int df = reader.docFreq(startTerm);
- int numVariants = 0;
- int totalVariantDocFreqs = 0;
- BytesRef possibleMatch;
- BoostAttribute boostAtt =
+ SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
+ //store the df so all variants use same idf
+ int df = reader.docFreq(startTerm);
+ int numVariants = 0;
+ int totalVariantDocFreqs = 0;
+ BytesRef possibleMatch;
+ BoostAttribute boostAtt =
fe.attributes().addAttribute(BoostAttribute.class);
- while ((possibleMatch = fe.next()) != null) {
- numVariants++;
- totalVariantDocFreqs += fe.docFreq();
- float score = boostAtt.getBoost();
- if (variantsQ.size() < MAX_VARIANTS_PER_TERM || score > minScore) {
- ScoreTerm st = new ScoreTerm(new Term(startTerm.field(), BytesRef.deepCopyOf(possibleMatch)), score, startTerm);
- variantsQ.insertWithOverflow(st);
- minScore = variantsQ.top().score; // maintain minScore
+ while ((possibleMatch = fe.next()) != null) {
+ numVariants++;
+ totalVariantDocFreqs += fe.docFreq();
+ float score = boostAtt.getBoost();
+ if (variantsQ.size() < MAX_VARIANTS_PER_TERM || score > minScore) {
+ ScoreTerm st = new ScoreTerm(new Term(startTerm.field(), BytesRef.deepCopyOf(possibleMatch)), score, startTerm);
+ variantsQ.insertWithOverflow(st);
+ minScore = variantsQ.top().score; // maintain minScore
+ }
+ maxBoostAtt.setMaxNonCompetitiveBoost(variantsQ.size() >= MAX_VARIANTS_PER_TERM ? minScore : Float.NEGATIVE_INFINITY);
}
- maxBoostAtt.setMaxNonCompetitiveBoost(variantsQ.size() >= MAX_VARIANTS_PER_TERM ? minScore : Float.NEGATIVE_INFINITY);
- }
- if (numVariants > 0) {
- int avgDf = totalVariantDocFreqs / numVariants;
- if (df == 0)//no direct match we can use as df for all variants
- {
- df = avgDf; //use avg df of all variants
- }
+ if (numVariants > 0) {
+ int avgDf = totalVariantDocFreqs / numVariants;
+ if (df == 0)//no direct match we can use as df for all variants
+ {
+ df = avgDf; //use avg df of all variants
+ }
- // take the top variants (scored by edit distance) and reset the score
- // to include an IDF factor then add to the global queue for ranking
- // overall top query terms
- int size = variantsQ.size();
- for (int i = 0; i < size; i++) {
- ScoreTerm st = variantsQ.pop();
- st.score = (st.score * st.score) * sim.idf(df, corpusNumDocs);
- q.insertWithOverflow(st);
+ // take the top variants (scored by edit distance) and reset the score
+ // to include an IDF factor then add to the global queue for ranking
+ // overall top query terms
+ int size = variantsQ.size();
+ for (int i = 0; i < size; i++) {
+ ScoreTerm st = variantsQ.pop();
+ st.score = (st.score * st.score) * sim.idf(df, corpusNumDocs);
+ q.insertWithOverflow(st);
+ }
}
}
}
+ ts.end();
}
- ts.end();
- ts.close();
}
@Override
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Mon Oct 7 06:24:25 2013
@@ -352,9 +352,8 @@ public class AnalyzingInfixSuggester ext
occur = BooleanClause.Occur.SHOULD;
}
- try {
+ try (TokenStream ts = queryAnalyzer.tokenStream("", new StringReader(key.toString()))) {
//long t0 = System.currentTimeMillis();
- TokenStream ts = queryAnalyzer.tokenStream("", new StringReader(key.toString()));
ts.reset();
final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
@@ -464,40 +463,39 @@ public class AnalyzingInfixSuggester ext
* result is set on each {@link
* LookupResult#highlightKey} member. */
protected Object highlight(String text, Set<String> matchedTokens, String prefixToken) throws IOException {
- TokenStream ts = queryAnalyzer.tokenStream("text", new StringReader(text));
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- ts.reset();
- StringBuilder sb = new StringBuilder();
- int upto = 0;
- while (ts.incrementToken()) {
- String token = termAtt.toString();
- int startOffset = offsetAtt.startOffset();
- int endOffset = offsetAtt.endOffset();
- if (upto < startOffset) {
- addNonMatch(sb, text.substring(upto, startOffset));
- upto = startOffset;
- } else if (upto > startOffset) {
- continue;
+ try (TokenStream ts = queryAnalyzer.tokenStream("text", new StringReader(text))) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+ ts.reset();
+ StringBuilder sb = new StringBuilder();
+ int upto = 0;
+ while (ts.incrementToken()) {
+ String token = termAtt.toString();
+ int startOffset = offsetAtt.startOffset();
+ int endOffset = offsetAtt.endOffset();
+ if (upto < startOffset) {
+ addNonMatch(sb, text.substring(upto, startOffset));
+ upto = startOffset;
+ } else if (upto > startOffset) {
+ continue;
+ }
+
+ if (matchedTokens.contains(token)) {
+ // Token matches.
+ addWholeMatch(sb, text.substring(startOffset, endOffset), token);
+ upto = endOffset;
+ } else if (prefixToken != null && token.startsWith(prefixToken)) {
+ addPrefixMatch(sb, text.substring(startOffset, endOffset), token, prefixToken);
+ upto = endOffset;
+ }
}
-
- if (matchedTokens.contains(token)) {
- // Token matches.
- addWholeMatch(sb, text.substring(startOffset, endOffset), token);
- upto = endOffset;
- } else if (prefixToken != null && token.startsWith(prefixToken)) {
- addPrefixMatch(sb, text.substring(startOffset, endOffset), token, prefixToken);
- upto = endOffset;
+ ts.end();
+ int endOffset = offsetAtt.endOffset();
+ if (upto < endOffset) {
+ addNonMatch(sb, text.substring(upto));
}
+ return sb.toString();
}
- ts.end();
- int endOffset = offsetAtt.endOffset();
- if (upto < endOffset) {
- addNonMatch(sb, text.substring(upto));
- }
- ts.close();
-
- return sb.toString();
}
/** Called while highlighting a single result, to append a
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1529770&r1=1529769&r2=1529770&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Mon Oct 7 06:24:25 2013
@@ -827,14 +827,15 @@ public class AnalyzingSuggester extends
}
final Set<IntsRef> toFiniteStrings(final BytesRef surfaceForm, final TokenStreamToAutomaton ts2a) throws IOException {
- // Analyze surface form:
- TokenStream ts = indexAnalyzer.tokenStream("", surfaceForm.utf8ToString());
-
- // Create corresponding automaton: labels are bytes
- // from each analyzed token, with byte 0 used as
- // separator between tokens:
- Automaton automaton = ts2a.toAutomaton(ts);
- ts.close();
+ // Analyze surface form:
+ Automaton automaton = null;
+ try (TokenStream ts = indexAnalyzer.tokenStream("", surfaceForm.utf8ToString())) {
+
+ // Create corresponding automaton: labels are bytes
+ // from each analyzed token, with byte 0 used as
+ // separator between tokens:
+ automaton = ts2a.toAutomaton(ts);
+ }
replaceSep(automaton);
automaton = convertAutomaton(automaton);
@@ -854,9 +855,10 @@ public class AnalyzingSuggester extends
final Automaton toLookupAutomaton(final CharSequence key) throws IOException {
// TODO: is there a Reader from a CharSequence?
// Turn tokenstream into automaton:
- TokenStream ts = queryAnalyzer.tokenStream("", key.toString());
- Automaton automaton = (getTokenStreamToAutomaton()).toAutomaton(ts);
- ts.close();
+ Automaton automaton = null;
+ try (TokenStream ts = queryAnalyzer.tokenStream("", key.toString())) {
+ automaton = (getTokenStreamToAutomaton()).toAutomaton(ts);
+ }
// TODO: we could use the end offset to "guess"
// whether the final token was a partial token; this