You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2015/07/02 17:07:42 UTC
svn commit: r1688845 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/
lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/
lucene/analysis/morfologik/src/j...
Author: uschindler
Date: Thu Jul 2 15:07:40 2015
New Revision: 1688845
URL: http://svn.apache.org/r1688845
Log:
Merged revision(s) 1688830 from lucene/dev/trunk:
LUCENE-6653, LUCENE-6652: Refactor TermToBytesRefAttribute; add oal.analysis.tokenattributes.BytesTermAttribute; remove code duplication in tests
Added:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.java
- copied unchanged from r1688830, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
- copied unchanged from r1688830, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
Removed:
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java
lucene/dev/branches/branch_5x/lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java
lucene/dev/branches/branch_5x/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttributeImpl.java
lucene/dev/branches/branch_5x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_5x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
lucene/dev/branches/branch_5x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
lucene/dev/branches/branch_5x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/codecs/src/test/org/apache/lucene/codecs/autoprefix/TestAutoPrefixTerms.java
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/Field.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java
lucene/dev/branches/branch_5x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/branches/branch_5x/lucene/memory/ (props changed)
lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_5x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java
lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java
lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java
lucene/dev/branches/branch_5x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/search/TokenStreamToTermAutomatonQuery.java
lucene/dev/branches/branch_5x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
lucene/dev/branches/branch_5x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionTokenStream.java
lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/contrib/ (props changed)
lucene/dev/branches/branch_5x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/CollationField.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/TextField.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Thu Jul 2 15:07:40 2015
@@ -77,6 +77,10 @@ New Features
* LUCENE-6632: Geo3D: Compute circle planes more accurately.
(Karl Wright via David Smiley)
+* LUCENE-6653: Added general purpose BytesTermAttribute to basic token
+ attributes package that can be used for TokenStreams that solely produce
+ binary terms. (Uwe Schindler)
+
API Changes
* LUCENE-6508: Simplify Lock api, there is now just
@@ -259,6 +263,9 @@ Test Framework
environments (e.g., read-only dirs). If tests are running without a security
manager, an assume cancels test execution automatically. (Uwe Schindler)
+* LUCENE-6652: Removed lots of useless Byte(s)TermAttributes all over test
+ infrastructure. (Uwe Schindler)
+
Changes in Backwards Compatibility Policy
* LUCENE-6553: The iterator returned by the LeafReader.postings method now
@@ -269,6 +276,11 @@ Changes in Backwards Compatibility Polic
DiversifiedTopDocsCollector can be used instead with a maximum number of hits
per key equal to 1. (Adrien Grand)
+* LUCENE-6653: The workflow for consuming the TermToBytesRefAttribute was changed:
+ getBytesRef() now does all work and is called on each token, fillBytesRef()
+ was removed. The implementation is free to reuse the internal BytesRef
+ or return a new one on each call. (Uwe Schindler)
+
======================= Lucene 5.2.1 =======================
Bug Fixes
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java Thu Jul 2 15:07:40 2015
@@ -40,11 +40,12 @@ public class CollatedTermAttributeImpl e
}
@Override
- public void fillBytesRef() {
- BytesRef bytes = getBytesRef();
- bytes.bytes = collator.getCollationKey(toString()).toByteArray();
- bytes.offset = 0;
- bytes.length = bytes.bytes.length;
+ public BytesRef getBytesRef() {
+ final BytesRef ref = this.builder.get();
+ ref.bytes = collator.getCollationKey(toString()).toByteArray();
+ ref.offset = 0;
+ ref.length = ref.bytes.length;
+ return ref;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java Thu Jul 2 15:07:40 2015
@@ -45,11 +45,12 @@ public class ICUCollatedTermAttributeImp
}
@Override
- public void fillBytesRef() {
- BytesRef bytes = getBytesRef();
+ public BytesRef getBytesRef() {
collator.getRawCollationKey(toString(), key);
- bytes.bytes = key.bytes;
- bytes.offset = 0;
- bytes.length = key.size;
+ final BytesRef ref = this.builder.get();
+ ref.bytes = key.bytes;
+ ref.offset = 0;
+ ref.length = key.size;
+ return ref;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttributeImpl.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttributeImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttributeImpl.java Thu Jul 2 15:07:40 2015
@@ -96,9 +96,4 @@ public class MorphosyntacticTagsAttribut
this.copyTo(cloned);
return cloned;
}
-
- @Override
- public String toString() {
- return tags == null ? "<no tags>" : tags.toString();
- }
}
Modified: lucene/dev/branches/branch_5x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/dev/branches/branch_5x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Thu Jul 2 15:07:40 2015
@@ -87,7 +87,7 @@ public class ReadTokensTask extends Perf
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
while(stream.incrementToken()) {
- termAtt.fillBytesRef();
+ termAtt.getBytesRef();
tokenCount++;
}
stream.end();
Modified: lucene/dev/branches/branch_5x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/branches/branch_5x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Thu Jul 2 15:07:40 2015
@@ -976,8 +976,6 @@ public class TestPerfTasksLogic extends
assertTrue(ts2.incrementToken());
BytesRef bytes1 = termAtt1.getBytesRef();
BytesRef bytes2 = termAtt2.getBytesRef();
- termAtt1.fillBytesRef();
- termAtt2.fillBytesRef();
assertEquals(bytes1, bytes2);
assertFalse(ts1.incrementToken());
assertFalse(ts2.incrementToken());
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/test/org/apache/lucene/codecs/autoprefix/TestAutoPrefixTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/test/org/apache/lucene/codecs/autoprefix/TestAutoPrefixTerms.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/test/org/apache/lucene/codecs/autoprefix/TestAutoPrefixTerms.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/test/org/apache/lucene/codecs/autoprefix/TestAutoPrefixTerms.java Thu Jul 2 15:07:40 2015
@@ -29,8 +29,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@@ -58,7 +56,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@@ -234,7 +231,7 @@ public class TestAutoPrefixTerms extends
for(Integer term : terms) {
Document doc = new Document();
- doc.add(new BinaryField("field", intToBytes(term)));
+ doc.add(newStringField("field", intToBytes(term), Field.Store.NO));
doc.add(new NumericDocValuesField("field", term));
w.addDocument(doc);
}
@@ -506,78 +503,6 @@ public class TestAutoPrefixTerms extends
dir.close();
}
- static final class BinaryTokenStream extends TokenStream {
- private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
- private boolean available = true;
-
- public BinaryTokenStream(BytesRef bytes) {
- bytesAtt.setBytesRef(bytes);
- }
-
- @Override
- public boolean incrementToken() {
- if (available) {
- clearAttributes();
- available = false;
- return true;
- }
- return false;
- }
-
- @Override
- public void reset() {
- available = true;
- }
-
- public interface ByteTermAttribute extends TermToBytesRefAttribute {
- void setBytesRef(BytesRef bytes);
- }
-
- public static class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute,TermToBytesRefAttribute {
- private BytesRef bytes;
-
- @Override
- public void fillBytesRef() {
- // no-op: the bytes was already filled by our owner's incrementToken
- }
-
- @Override
- public BytesRef getBytesRef() {
- return bytes;
- }
-
- @Override
- public void setBytesRef(BytesRef bytes) {
- this.bytes = bytes;
- }
-
- @Override
- public void clear() {}
-
- @Override
- public void copyTo(AttributeImpl target) {
- ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
- other.bytes = bytes;
- }
- }
- }
-
- /** Basically a StringField that accepts binary term. */
- private static class BinaryField extends Field {
-
- final static FieldType TYPE;
- static {
- TYPE = new FieldType(StringField.TYPE_NOT_STORED);
- // Necessary so our custom tokenStream is used by Field.tokenStream:
- TYPE.setTokenized(true);
- TYPE.freeze();
- }
-
- public BinaryField(String name, BytesRef value) {
- super(name, new BinaryTokenStream(value), TYPE);
- }
- }
-
/** Helper class to ensure auto-prefix terms 1) never overlap one another, and 2) are used when they should be. */
private static class VerifyAutoPrefixTerms {
final FixedBitSet allHits;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java Thu Jul 2 15:07:40 2015
@@ -158,17 +158,13 @@ public final class NumericTokenStream ex
@Override
public BytesRef getBytesRef() {
- return bytes.get();
- }
-
- @Override
- public void fillBytesRef() {
assert valueSize == 64 || valueSize == 32;
if (valueSize == 64) {
NumericUtils.longToPrefixCoded(value, shift, bytes);
} else {
NumericUtils.intToPrefixCoded((int) value, shift, bytes);
}
+ return bytes.get();
}
@Override
@@ -201,8 +197,7 @@ public final class NumericTokenStream ex
@Override
public void reflectWith(AttributeReflector reflector) {
- fillBytesRef();
- reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes.toBytesRef());
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", getBytesRef());
reflector.reflect(NumericTermAttribute.class, "shift", shift);
reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue());
reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize);
@@ -219,7 +214,7 @@ public final class NumericTokenStream ex
NumericTermAttributeImpl t = (NumericTermAttributeImpl)super.clone();
// Do a deep clone
t.bytes = new BytesRefBuilder();
- t.bytes.copyBytes(bytes.get());
+ t.bytes.copyBytes(getBytesRef());
return t;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java Thu Jul 2 15:07:40 2015
@@ -105,8 +105,6 @@ public class TokenStreamToAutomaton {
final PositionLengthAttribute posLengthAtt = in.addAttribute(PositionLengthAttribute.class);
final OffsetAttribute offsetAtt = in.addAttribute(OffsetAttribute.class);
- final BytesRef term = termBytesAtt.getBytesRef();
-
in.reset();
// Only temporarily holds states ahead of our current
@@ -157,8 +155,7 @@ public class TokenStreamToAutomaton {
final int endPos = pos + posLengthAtt.getPositionLength();
- termBytesAtt.fillBytesRef();
- final BytesRef termUTF8 = changeToken(term);
+ final BytesRef termUTF8 = changeToken(termBytesAtt.getBytesRef());
int[] termUnicode = null;
final Position endPosData = positions.get(endPos);
if (endPosData.arriving == -1) {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Thu Jul 2 15:07:40 2015
@@ -33,6 +33,9 @@ public class CharTermAttributeImpl exten
private char[] termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
private int termLength = 0;
+ /** May be used by subclasses to convert to different charsets / encodings for implementing {@link #getBytesRef()}. */
+ protected BytesRefBuilder builder = new BytesRefBuilder();
+
/** Initialize this attribute with empty term text */
public CharTermAttributeImpl() {}
@@ -83,16 +86,11 @@ public class CharTermAttributeImpl exten
}
// *** TermToBytesRefAttribute interface ***
- private BytesRefBuilder bytes = new BytesRefBuilder();
-
- @Override
- public void fillBytesRef() {
- bytes.copyChars(termBuffer, 0, termLength);
- }
@Override
public BytesRef getBytesRef() {
- return bytes.get();
+ builder.copyChars(termBuffer, 0, termLength);
+ return builder.get();
}
// *** CharSequence interface ***
@@ -228,8 +226,8 @@ public class CharTermAttributeImpl exten
// Do a deep clone
t.termBuffer = new char[this.termLength];
System.arraycopy(this.termBuffer, 0, t.termBuffer, 0, this.termLength);
- t.bytes = new BytesRefBuilder();
- t.bytes.copyBytes(bytes.get());
+ t.builder = new BytesRefBuilder();
+ t.builder.copyBytes(builder.get());
return t;
}
@@ -271,8 +269,7 @@ public class CharTermAttributeImpl exten
@Override
public void reflectWith(AttributeReflector reflector) {
reflector.reflect(CharTermAttribute.class, "term", toString());
- fillBytesRef();
- reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes.toBytesRef());
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", getBytesRef());
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java Thu Jul 2 15:07:40 2015
@@ -24,17 +24,12 @@ import org.apache.lucene.util.BytesRef;
* This attribute is requested by TermsHashPerField to index the contents.
* This attribute can be used to customize the final byte[] encoding of terms.
* <p>
- * Consumers of this attribute call {@link #getBytesRef()} up-front, and then
- * invoke {@link #fillBytesRef()} for each term. Example:
+ * Consumers of this attribute call {@link #getBytesRef()} for each term. Example:
* <pre class="prettyprint">
* final TermToBytesRefAttribute termAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
- * final BytesRef bytes = termAtt.getBytesRef();
*
* while (tokenStream.incrementToken() {
- *
- * // you must call termAtt.fillBytesRef() before doing something with the bytes.
- * // this encodes the term value (internally it might be a char[], etc) into the bytes.
- * int hashCode = termAtt.fillBytesRef();
+ * final BytesRef bytes = termAtt.getBytesRef();
*
* if (isInteresting(bytes)) {
*
@@ -42,27 +37,21 @@ import org.apache.lucene.util.BytesRef;
* // you should make a copy if you need persistent access to the bytes, otherwise they will
* // be rewritten across calls to incrementToken()
*
- * doSomethingWith(new BytesRef(bytes));
+ * doSomethingWith(BytesRef.deepCopyOf(bytes));
* }
* }
* ...
* </pre>
- * @lucene.experimental This is a very expert API, please use
- * {@link CharTermAttributeImpl} and its implementation of this method
- * for UTF-8 terms.
+ * @lucene.internal This is a very expert and internal API, please use
+ * {@link CharTermAttribute} and its implementation for UTF-8 terms; to
+ * index binary terms, use {@link BytesTermAttribute} and its implementation.
*/
public interface TermToBytesRefAttribute extends Attribute {
-
- /**
- * Updates the bytes {@link #getBytesRef()} to contain this term's
- * final encoding.
- */
- public void fillBytesRef();
/**
- * Retrieve this attribute's BytesRef. The bytes are updated
- * from the current term when the consumer calls {@link #fillBytesRef()}.
- * @return this Attributes internal BytesRef.
+ * Retrieve this attribute's BytesRef. The bytes are updated from the current term.
+ * The implementation may return a new instance or keep the previous one.
+ * @return a BytesRef to be indexed (only stays valid until token stream gets incremented)
*/
public BytesRef getBytesRef();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/Field.java Thu Jul 2 15:07:40 2015
@@ -23,16 +23,14 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.FieldType.NumericType;
-import org.apache.lucene.index.FieldInvertState; // javadocs
import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter; // javadocs
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
-import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.BytesRef;
/**
@@ -568,71 +566,47 @@ public class Field implements IndexableF
}
private static final class BinaryTokenStream extends TokenStream {
- private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
-
- // Do not init this to true, becase caller must first call reset:
- private boolean available;
-
- public BinaryTokenStream() {
+ private final BytesTermAttribute bytesAtt = addAttribute(BytesTermAttribute.class);
+ private boolean used = true;
+ private BytesRef value;
+
+ /** Creates a new TokenStream that returns a BytesRef as single token.
+ * <p>Warning: Does not initialize the value, you must call
+ * {@link #setValue(BytesRef)} afterwards!
+ */
+ BinaryTokenStream() {
}
public void setValue(BytesRef value) {
- bytesAtt.setBytesRef(value);
+ this.value = value;
}
@Override
public boolean incrementToken() {
- if (available) {
- clearAttributes();
- available = false;
- return true;
+ if (used) {
+ return false;
}
- return false;
+ clearAttributes();
+ bytesAtt.setBytesRef(value);
+ used = true;
+ return true;
}
@Override
public void reset() {
- available = true;
+ used = false;
}
-
- public interface ByteTermAttribute extends TermToBytesRefAttribute {
- public void setBytesRef(BytesRef bytes);
- }
-
- public static class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute, TermToBytesRefAttribute {
- private BytesRef bytes;
-
- @Override
- public void fillBytesRef() {
- // no-op: the bytes was already filled by our owner's incrementToken
- }
-
- @Override
- public BytesRef getBytesRef() {
- return bytes;
- }
- @Override
- public void setBytesRef(BytesRef bytes) {
- this.bytes = bytes;
- }
-
- @Override
- public void clear() {
- }
-
- @Override
- public void copyTo(AttributeImpl target) {
- ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
- other.bytes = bytes;
- }
+ @Override
+ public void close() {
+ value = null;
}
}
private static final class StringTokenStream extends TokenStream {
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
- private boolean used = false;
+ private boolean used = true;
private String value = null;
/** Creates a new TokenStream that returns a String as single token.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java Thu Jul 2 15:07:40 2015
@@ -36,7 +36,6 @@ abstract class TermsHashPerField impleme
protected final DocumentsWriterPerThread.DocState docState;
protected final FieldInvertState fieldState;
TermToBytesRefAttribute termAtt;
- BytesRef termBytesRef;
// Copied from our perThread
final IntBlockPool intPool;
@@ -145,13 +144,10 @@ abstract class TermsHashPerField impleme
* entry point (for first TermsHash); postings use this
* API. */
void add() throws IOException {
-
- termAtt.fillBytesRef();
-
// We are first in the chain so we must "intern" the
// term text into textStart address
// Get the text & hash of this term.
- int termID = bytesHash.add(termBytesRef);
+ int termID = bytesHash.add(termAtt.getBytesRef());
//System.out.println("add term=" + termBytesRef.utf8ToString() + " doc=" + docState.docID + " termID=" + termID);
@@ -292,10 +288,6 @@ abstract class TermsHashPerField impleme
* document. */
boolean start(IndexableField field, boolean first) {
termAtt = fieldState.termAttribute;
- // EmptyTokenStream can have null term att
- if (termAtt != null) {
- termBytesRef = termAtt.getBytesRef();
- }
if (nextPerField != null) {
doNextCall = nextPerField.start(field, first);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java Thu Jul 2 15:07:40 2015
@@ -261,15 +261,13 @@ public class QueryBuilder {
*/
private Query analyzeTerm(String field, TokenStream stream) throws IOException {
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
stream.reset();
if (!stream.incrementToken()) {
throw new AssertionError();
}
- termAtt.fillBytesRef();
- return newTermQuery(new Term(field, BytesRef.deepCopyOf(bytes)));
+ return newTermQuery(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
}
/**
@@ -279,12 +277,10 @@ public class QueryBuilder {
BooleanQuery q = newBooleanQuery(true);
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
stream.reset();
while (stream.incrementToken()) {
- termAtt.fillBytesRef();
- Query currentQuery = newTermQuery(new Term(field, BytesRef.deepCopyOf(bytes)));
+ Query currentQuery = newTermQuery(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
@@ -299,13 +295,11 @@ public class QueryBuilder {
Query currentQuery = null;
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
-
PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
stream.reset();
while (stream.incrementToken()) {
- termAtt.fillBytesRef();
+ BytesRef bytes = termAtt.getBytesRef();
if (posIncrAtt.getPositionIncrement() == 0) {
if (!(currentQuery instanceof BooleanQuery)) {
Query t = currentQuery;
@@ -333,15 +327,12 @@ public class QueryBuilder {
pq.setSlop(slop);
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
-
PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
int position = -1;
stream.reset();
while (stream.incrementToken()) {
- termAtt.fillBytesRef();
-
+ BytesRef bytes = termAtt.getBytesRef();
if (enablePositionIncrements) {
position += posIncrAtt.getPositionIncrement();
pq.add(new Term(field, BytesRef.deepCopyOf(bytes)), position);
@@ -361,7 +352,6 @@ public class QueryBuilder {
mpq.setSlop(slop);
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
int position = -1;
@@ -369,7 +359,6 @@ public class QueryBuilder {
List<Term> multiTerms = new ArrayList<>();
stream.reset();
while (stream.incrementToken()) {
- termAtt.fillBytesRef();
int positionIncrement = posIncrAtt.getPositionIncrement();
if (positionIncrement > 0 && multiTerms.size() > 0) {
@@ -381,7 +370,7 @@ public class QueryBuilder {
multiTerms.clear();
}
position += positionIncrement;
- multiTerms.add(new Term(field, BytesRef.deepCopyOf(bytes)));
+ multiTerms.add(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
}
if (enablePositionIncrements) {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Thu Jul 2 15:07:40 2015
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.analysis.NumericTokenStream.NumericTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
@@ -40,14 +39,12 @@ public class TestNumericTokenStream exte
assertNotNull(typeAtt);
final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
assertNotNull(numericAtt);
- final BytesRef bytes = bytesAtt.getBytesRef();
stream.reset();
assertEquals(64, numericAtt.getValueSize());
for (int shift=0; shift<64; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
- bytesAtt.fillBytesRef();
- assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytes));
+ assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytesAtt.getBytesRef()));
assertEquals("Term raw value is incorrectly encoded", lvalue & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
@@ -65,14 +62,12 @@ public class TestNumericTokenStream exte
assertNotNull(typeAtt);
final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
assertNotNull(numericAtt);
- final BytesRef bytes = bytesAtt.getBytesRef();
stream.reset();
assertEquals(32, numericAtt.getValueSize());
for (int shift=0; shift<32; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
- bytesAtt.fillBytesRef();
- assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytes));
+ assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytesAtt.getBytesRef()));
assertEquals("Term raw value is incorrectly encoded", ((long) ivalue) & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
@@ -123,6 +118,7 @@ public class TestNumericTokenStream exte
public void testAttributeClone() throws Exception {
NumericTermAttributeImpl att = new NumericTermAttributeImpl();
+ att.init(1234L, 64, 8, 0); // set some value, to make getBytesRef() work
NumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att);
assertNotSame(att.getBytesRef(), copy.getBytesRef());
NumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java Thu Jul 2 15:07:40 2015
@@ -109,11 +109,6 @@ public class Test2BTerms extends LuceneT
private final static class MyTermAttributeImpl extends AttributeImpl implements TermToBytesRefAttribute {
@Override
- public void fillBytesRef() {
- // no-op: the bytes was already filled by our owner's incrementToken
- }
-
- @Override
public BytesRef getBytesRef() {
return bytes;
}
@@ -123,17 +118,8 @@ public class Test2BTerms extends LuceneT
}
@Override
- public boolean equals(Object other) {
- return other == this;
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
public void copyTo(AttributeImpl target) {
+ throw new UnsupportedOperationException();
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java Thu Jul 2 15:07:40 2015
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -38,7 +37,6 @@ public class TestBinaryTerms extends Luc
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
BytesRef bytes = new BytesRef(2);
- BinaryTokenStream tokenStream = new BinaryTokenStream(bytes);
for (int i = 0; i < 256; i++) {
bytes.bytes[0] = (byte) i;
@@ -47,8 +45,8 @@ public class TestBinaryTerms extends Luc
Document doc = new Document();
FieldType customType = new FieldType();
customType.setStored(true);
- doc.add(new Field("id", "" + i, customType));
- doc.add(new TextField("bytes", tokenStream));
+ doc.add(newField("id", "" + i, customType));
+ doc.add(newStringField("bytes", bytes, Field.Store.NO));
iw.addDocument(doc);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Thu Jul 2 15:07:40 2015
@@ -49,14 +49,13 @@ public class TestLongPostings extends Lu
}
try (TokenStream ts = a.tokenStream("foo", s)) {
final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
- final BytesRef termBytes = termAtt.getBytesRef();
ts.reset();
int count = 0;
boolean changed = false;
while(ts.incrementToken()) {
- termAtt.fillBytesRef();
+ final BytesRef termBytes = termAtt.getBytesRef();
if (count == 0 && !termBytes.utf8ToString().equals(s)) {
// The value was changed during analysis. Keep iterating so the
// tokenStream is exhausted.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java Thu Jul 2 15:07:40 2015
@@ -23,17 +23,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
@@ -94,78 +89,6 @@ public class TestPrefixQuery extends Luc
directory.close();
}
- static final class BinaryTokenStream extends TokenStream {
- private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
- private boolean available = true;
-
- public BinaryTokenStream(BytesRef bytes) {
- bytesAtt.setBytesRef(bytes);
- }
-
- @Override
- public boolean incrementToken() {
- if (available) {
- clearAttributes();
- available = false;
- return true;
- }
- return false;
- }
-
- @Override
- public void reset() {
- available = true;
- }
-
- public interface ByteTermAttribute extends TermToBytesRefAttribute {
- public void setBytesRef(BytesRef bytes);
- }
-
- public static class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute,TermToBytesRefAttribute {
- private BytesRef bytes;
-
- @Override
- public void fillBytesRef() {
- // no-op: the bytes was already filled by our owner's incrementToken
- }
-
- @Override
- public BytesRef getBytesRef() {
- return bytes;
- }
-
- @Override
- public void setBytesRef(BytesRef bytes) {
- this.bytes = bytes;
- }
-
- @Override
- public void clear() {}
-
- @Override
- public void copyTo(AttributeImpl target) {
- ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
- other.bytes = bytes;
- }
- }
- }
-
- /** Basically a StringField that accepts binary term. */
- private static class BinaryField extends Field {
-
- final static FieldType TYPE;
- static {
- TYPE = new FieldType(StringField.TYPE_NOT_STORED);
- // Necessary so our custom tokenStream is used by Field.tokenStream:
- TYPE.setTokenized(true);
- TYPE.freeze();
- }
-
- public BinaryField(String name, BytesRef value) {
- super(name, new BinaryTokenStream(value), TYPE);
- }
- }
-
public void testRandomBinaryPrefix() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
@@ -182,7 +105,7 @@ public class TestPrefixQuery extends Luc
Collections.shuffle(termsList, random());
for(BytesRef term : termsList) {
Document doc = new Document();
- doc.add(new BinaryField("field", term));
+ doc.add(newStringField("field", term, Field.Store.NO));
w.addDocument(doc);
}
Modified: lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Thu Jul 2 15:07:40 2015
@@ -171,14 +171,11 @@ public abstract class AbstractTestCase e
try (TokenStream tokenStream = analyzer.tokenStream(field, text)) {
TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);
-
- BytesRef bytesRef = termAttribute.getBytesRef();
-
+
tokenStream.reset();
while (tokenStream.incrementToken()) {
- termAttribute.fillBytesRef();
- bytesRefs.add(BytesRef.deepCopyOf(bytesRef));
+ bytesRefs.add(BytesRef.deepCopyOf(termAttribute.getBytesRef()));
}
tokenStream.end();
Modified: lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Jul 2 15:07:40 2015
@@ -453,18 +453,16 @@ public class MemoryIndex {
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
PayloadAttribute payloadAtt = storePayloads ? stream.addAttribute(PayloadAttribute.class) : null;
- BytesRef ref = termAtt.getBytesRef();
stream.reset();
while (stream.incrementToken()) {
- termAtt.fillBytesRef();
// if (DEBUG) System.err.println("token='" + term + "'");
numTokens++;
final int posIncr = posIncrAttribute.getPositionIncrement();
if (posIncr == 0)
numOverlapTokens++;
pos += posIncr;
- int ord = terms.add(ref);
+ int ord = terms.add(termAtt.getBytesRef());
if (ord < 0) {
ord = (-ord) - 1;
postingsWriter.reset(sliceArray.end[ord]);
Modified: lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Thu Jul 2 15:07:40 2015
@@ -602,15 +602,14 @@ public abstract class QueryParserBase ex
source.reset();
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
if (!source.incrementToken())
throw new IllegalArgumentException("analyzer returned no terms for multiTerm term: " + part);
- termAtt.fillBytesRef();
+ BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef());
if (source.incrementToken())
throw new IllegalArgumentException("analyzer returned too many terms for multiTerm term: " + part);
source.end();
- return BytesRef.deepCopyOf(bytes);
+ return bytes;
} catch (IOException e) {
throw new RuntimeException("Error analyzing multiTerm term: " + part, e);
}
Modified: lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java Thu Jul 2 15:07:40 2015
@@ -53,11 +53,9 @@ public class SpanOrTermsBuilder extends
try (TokenStream ts = analyzer.tokenStream(fieldName, value)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
ts.reset();
while (ts.incrementToken()) {
- termAtt.fillBytesRef();
- SpanTermQuery stq = new SpanTermQuery(new Term(fieldName, BytesRef.deepCopyOf(bytes)));
+ SpanTermQuery stq = new SpanTermQuery(new Term(fieldName, BytesRef.deepCopyOf(termAtt.getBytesRef())));
clausesList.add(stq);
}
ts.end();
Modified: lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java Thu Jul 2 15:07:40 2015
@@ -56,11 +56,9 @@ public class TermsFilterBuilder implemen
try (TokenStream ts = analyzer.tokenStream(fieldName, text)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
ts.reset();
while (ts.incrementToken()) {
- termAtt.fillBytesRef();
- terms.add(BytesRef.deepCopyOf(bytes));
+ terms.add(BytesRef.deepCopyOf(termAtt.getBytesRef()));
}
ts.end();
}
Modified: lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java Thu Jul 2 15:07:40 2015
@@ -54,11 +54,9 @@ public class TermsQueryBuilder implement
try (TokenStream ts = analyzer.tokenStream(fieldName, text)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
- BytesRef bytes = termAtt.getBytesRef();
ts.reset();
while (ts.incrementToken()) {
- termAtt.fillBytesRef();
- term = new Term(fieldName, BytesRef.deepCopyOf(bytes));
+ term = new Term(fieldName, BytesRef.deepCopyOf(termAtt.getBytesRef()));
bq.add(new BooleanClause(new TermQuery(term), BooleanClause.Occur.SHOULD));
}
ts.end();
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/search/TokenStreamToTermAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/search/TokenStreamToTermAutomatonQuery.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/search/TokenStreamToTermAutomatonQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/search/TokenStreamToTermAutomatonQuery.java Thu Jul 2 15:07:40 2015
@@ -61,8 +61,6 @@ public class TokenStreamToTermAutomatonQ
final PositionLengthAttribute posLengthAtt = in.addAttribute(PositionLengthAttribute.class);
final OffsetAttribute offsetAtt = in.addAttribute(OffsetAttribute.class);
- final BytesRef term = termBytesAtt.getBytesRef();
-
in.reset();
TermAutomatonQuery query = new TermAutomatonQuery(field);
@@ -93,7 +91,7 @@ public class TokenStreamToTermAutomatonQ
state = query.createState();
}
- termBytesAtt.fillBytesRef();
+ BytesRef term = termBytesAtt.getBytesRef();
//System.out.println(pos + "-" + endPos + ": " + term.utf8ToString() + ": posInc=" + posInc);
if (term.length == 1 && term.bytes[term.offset] == (byte) '*') {
query.addAnyTransition(pos, endPos);
Modified: lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java Thu Jul 2 15:07:40 2015
@@ -20,11 +20,7 @@ package org.apache.lucene.spatial.prefix
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.util.Attribute;
-import org.apache.lucene.util.AttributeFactory;
-import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -37,85 +33,6 @@ import org.apache.lucene.util.BytesRefIt
*/
class BytesRefIteratorTokenStream extends TokenStream {
- // just a wrapper to prevent adding CharTermAttribute
- private static final class BRAttributeFactory extends AttributeFactory {
- private final AttributeFactory delegate;
-
- BRAttributeFactory(AttributeFactory delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
- if (CharTermAttribute.class.isAssignableFrom(attClass))
- throw new IllegalArgumentException(getClass() + " does not support CharTermAttribute.");
- return delegate.createAttributeInstance(attClass);
- }
- }
-
- private static final class BRTermToBytesRefAttributeImpl extends AttributeImpl
- implements TermToBytesRefAttribute {
- private final BytesRef bytes = new BytesRef();
-
- void setBytesRef(BytesRef inputBytes) {
- // shallow clone. this.bytesRef is final
- bytes.bytes = inputBytes.bytes;
- bytes.offset = inputBytes.offset;
- bytes.length = inputBytes.length;
- }
-
- @Override
- public void clear() {
- // we keep it untouched as it's fully controlled by the outer class.
- }
-
- @Override
- public void copyTo(AttributeImpl target) {
- final BRTermToBytesRefAttributeImpl a = (BRTermToBytesRefAttributeImpl) target;
- a.setBytesRef(BytesRef.deepCopyOf(bytes));
- }
-
- @Override
- public void fillBytesRef() {
- //nothing to do; it's populated by incrementToken
- }
-
- @Override
- public BytesRef getBytesRef() {
- return bytes;
- }
-
- @Override
- public BRTermToBytesRefAttributeImpl clone() {
- // super.clone won't work since we need a new BytesRef reference and it's nice to have it final. The superclass
- // has no state to copy anyway.
- final BRTermToBytesRefAttributeImpl clone = new BRTermToBytesRefAttributeImpl();
- clone.setBytesRef(BytesRef.deepCopyOf(bytes));
- return clone;
- }
-
- @Override
- public int hashCode() {
- return bytes.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) return true;
- if (obj == null) return false;
- if (getClass() != obj.getClass()) return false;
- BRTermToBytesRefAttributeImpl other = (BRTermToBytesRefAttributeImpl) obj;
- if (!bytes.equals(other.bytes)) return false;
- return true;
- }
- }
-
- public BytesRefIteratorTokenStream() {
- super(new BRAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY));
- addAttributeImpl(new BRTermToBytesRefAttributeImpl());//because non-public constructor
- bytesAtt = (BRTermToBytesRefAttributeImpl) addAttribute(TermToBytesRefAttribute.class);
- }
-
public BytesRefIterator getBytesRefIterator() {
return bytesIter;
}
@@ -129,7 +46,6 @@ class BytesRefIteratorTokenStream extend
public void reset() throws IOException {
if (bytesIter == null)
throw new IllegalStateException("call setBytesRefIterator() before usage");
- bytesAtt.getBytesRef().length = 0;
}
@Override
@@ -137,14 +53,12 @@ class BytesRefIteratorTokenStream extend
if (bytesIter == null)
throw new IllegalStateException("call setBytesRefIterator() before usage");
- // this will only clear all other attributes in this TokenStream
- clearAttributes();//TODO but there should be no "other" attributes
-
// get next
BytesRef bytes = bytesIter.next();
if (bytes == null) {
return false;
} else {
+ clearAttributes();
bytesAtt.setBytesRef(bytes);
//note: we don't bother setting posInc or type attributes. There's no point to it.
return true;
@@ -152,7 +66,7 @@ class BytesRefIteratorTokenStream extend
}
//members
- private final BRTermToBytesRefAttributeImpl bytesAtt;
+ private final BytesTermAttribute bytesAtt = addAttribute(BytesTermAttribute.class);
private BytesRefIterator bytesIter = null; // null means not initialized
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Thu Jul 2 15:07:40 2015
@@ -485,11 +485,10 @@ public class FreeTextSuggester extends L
// Run full analysis, but save only the
// last 1gram, last 2gram, etc.:
- BytesRef tokenBytes = termBytesAtt.getBytesRef();
int maxEndOffset = -1;
boolean sawRealToken = false;
while(ts.incrementToken()) {
- termBytesAtt.fillBytesRef();
+ BytesRef tokenBytes = termBytesAtt.getBytesRef();
sawRealToken |= tokenBytes.length > 0;
// TODO: this is somewhat iffy; today, ShingleFilter
// sets posLen to the gram count; maybe we should make
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionTokenStream.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionTokenStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionTokenStream.java Thu Jul 2 15:07:40 2015
@@ -58,7 +58,7 @@ public final class CompletionTokenStream
private final PayloadAttribute payloadAttr = addAttribute(PayloadAttribute.class);
private final PositionIncrementAttribute posAttr = addAttribute(PositionIncrementAttribute.class);
- private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
+ private final BytesRefBuilderTermAttribute bytesAtt = addAttribute(BytesRefBuilderTermAttribute.class);
private final TokenStream input;
final boolean preserveSep;
@@ -309,9 +309,7 @@ public final class CompletionTokenStream
/**
* Attribute providing access to the term builder and UTF-16 conversion
*/
- private interface ByteTermAttribute extends TermToBytesRefAttribute {
- // marker interface
-
+ private interface BytesRefBuilderTermAttribute extends TermToBytesRefAttribute {
/**
* Returns the builder from which the term is derived.
*/
@@ -326,20 +324,15 @@ public final class CompletionTokenStream
/**
* Custom attribute implementation for completion token stream
*/
- public static final class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute, TermToBytesRefAttribute {
+ public static final class BytesRefBuilderTermAttributeImpl extends AttributeImpl implements BytesRefBuilderTermAttribute, TermToBytesRefAttribute {
private final BytesRefBuilder bytes = new BytesRefBuilder();
- private CharsRefBuilder charsRef;
+ private transient CharsRefBuilder charsRef;
/**
* Sole constructor
* no-op
*/
- public ByteTermAttributeImpl() {
- }
-
- @Override
- public void fillBytesRef() {
- // does nothing - we change in place
+ public BytesRefBuilderTermAttributeImpl() {
}
@Override
@@ -359,11 +352,18 @@ public final class CompletionTokenStream
@Override
public void copyTo(AttributeImpl target) {
- ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
+ BytesRefBuilderTermAttributeImpl other = (BytesRefBuilderTermAttributeImpl) target;
other.bytes.copyBytes(bytes);
}
@Override
+ public AttributeImpl clone() {
+ BytesRefBuilderTermAttributeImpl other = new BytesRefBuilderTermAttributeImpl();
+ copyTo(other);
+ return other;
+ }
+
+ @Override
public CharSequence toUTF16() {
if (charsRef == null) {
charsRef = new CharsRefBuilder();
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Thu Jul 2 15:07:40 2015
@@ -51,7 +51,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
-import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LineFileDocs;
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java Thu Jul 2 15:07:40 2015
@@ -17,13 +17,11 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
/**
* TokenStream from a canned list of binary (BytesRef-based)
@@ -54,65 +52,11 @@ public final class CannedBinaryTokenStre
private final BinaryToken[] tokens;
private int upto = 0;
- private final BinaryTermAttribute termAtt = addAttribute(BinaryTermAttribute.class);
+ private final BytesTermAttribute termAtt = addAttribute(BytesTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLengthAtt = addAttribute(PositionLengthAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- /** An attribute extending {@link
- * TermToBytesRefAttribute} but exposing {@link
- * #setBytesRef} method. */
- public interface BinaryTermAttribute extends TermToBytesRefAttribute {
-
- /** Set the current binary value. */
- public void setBytesRef(BytesRef bytes);
- }
-
- /** Implementation for {@link BinaryTermAttribute}. */
- public final static class BinaryTermAttributeImpl extends AttributeImpl implements BinaryTermAttribute, TermToBytesRefAttribute {
- private final BytesRefBuilder bytes = new BytesRefBuilder();
-
- @Override
- public void fillBytesRef() {
- bytes.get(); // sets the length on the bytesref
- }
-
- @Override
- public BytesRef getBytesRef() {
- return bytes.get();
- }
-
- @Override
- public void setBytesRef(BytesRef bytes) {
- this.bytes.copyBytes(bytes);
- }
-
- @Override
- public void clear() {
- }
-
- @Override
- public boolean equals(Object other) {
- return other == this;
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
- public void copyTo(AttributeImpl target) {
- BinaryTermAttributeImpl other = (BinaryTermAttributeImpl) target;
- other.bytes.copyBytes(bytes);
- }
-
- @Override
- public BinaryTermAttributeImpl clone() {
- throw new UnsupportedOperationException();
- }
- }
-
public CannedBinaryTokenStream(BinaryToken... tokens) {
super();
this.tokens = tokens;
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java Thu Jul 2 15:07:40 2015
@@ -183,12 +183,10 @@ public abstract class CollationTestBase
String term = TestUtil.randomSimpleString(random());
try (TokenStream ts = analyzer.tokenStream("fake", term)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
ts.reset();
assertTrue(ts.incrementToken());
- termAtt.fillBytesRef();
// ensure we make a copy of the actual bytes too
- map.put(term, BytesRef.deepCopyOf(bytes));
+ map.put(term, BytesRef.deepCopyOf(termAtt.getBytesRef()));
assertFalse(ts.incrementToken());
ts.end();
}
@@ -205,11 +203,9 @@ public abstract class CollationTestBase
BytesRef expected = mapping.getValue();
try (TokenStream ts = analyzer.tokenStream("fake", term)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
ts.reset();
assertTrue(ts.incrementToken());
- termAtt.fillBytesRef();
- assertEquals(expected, bytes);
+ assertEquals(expected, termAtt.getBytesRef());
assertFalse(ts.incrementToken());
ts.end();
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java Thu Jul 2 15:07:40 2015
@@ -34,11 +34,11 @@ public class MockUTF16TermAttributeImpl
AttributeFactory.getStaticImplementation(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, MockUTF16TermAttributeImpl.class);
@Override
- public void fillBytesRef() {
- BytesRef bytes = getBytesRef();
- byte[] utf16 = toString().getBytes(StandardCharsets.UTF_16LE);
- bytes.bytes = utf16;
- bytes.offset = 0;
- bytes.length = utf16.length;
+ public BytesRef getBytesRef() {
+ final BytesRef ref = this.builder.get();
+ ref.bytes = toString().getBytes(StandardCharsets.UTF_16LE);
+ ref.offset = 0;
+ ref.length = ref.bytes.length;
+ return ref;
}
}
Modified: lucene/dev/branches/branch_5x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java Thu Jul 2 15:07:40 2015
@@ -253,16 +253,16 @@ public class ICUCollationField extends F
source.reset();
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
+
// we control the analyzer here: most errors are impossible
if (!source.incrementToken())
throw new IllegalArgumentException("analyzer returned no terms for text: " + text);
- termAtt.fillBytesRef();
+ BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef());
assert !source.incrementToken();
source.end();
- return BytesRef.deepCopyOf(bytes);
+ return bytes;
} catch (IOException e) {
throw new RuntimeException("Unable to analyze text: " + text, e);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Thu Jul 2 15:07:40 2015
@@ -148,13 +148,11 @@ public abstract class AnalysisRequestHan
try (TokenStream tokenStream = analyzer.tokenStream("", query)){
final Set<BytesRef> tokens = new HashSet<>();
final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
- final BytesRef bytes = bytesAtt.getBytesRef();
tokenStream.reset();
while (tokenStream.incrementToken()) {
- bytesAtt.fillBytesRef();
- tokens.add(BytesRef.deepCopyOf(bytes));
+ tokens.add(BytesRef.deepCopyOf(bytesAtt.getBytesRef()));
}
tokenStream.end();
@@ -246,7 +244,6 @@ public abstract class AnalysisRequestHan
final NamedList<Object> tokenNamedList = new SimpleOrderedMap<>();
final TermToBytesRefAttribute termAtt = token.getAttribute(TermToBytesRefAttribute.class);
BytesRef rawBytes = termAtt.getBytesRef();
- termAtt.fillBytesRef();
final String text = fieldType.indexedToReadable(rawBytes, new CharsRefBuilder()).toString();
tokenNamedList.add("text", text);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/CollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/CollationField.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/CollationField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/CollationField.java Thu Jul 2 15:07:40 2015
@@ -224,16 +224,14 @@ public class CollationField extends Fiel
try (TokenStream source = analyzer.tokenStream(field, text)) {
source.reset();
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
// we control the analyzer here: most errors are impossible
if (!source.incrementToken())
throw new IllegalArgumentException("analyzer returned no terms for text: " + text);
- termAtt.fillBytesRef();
+ BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef());
assert !source.incrementToken();
-
source.end();
- return BytesRef.deepCopyOf(bytes);
+ return bytes;
} catch (IOException e) {
throw new RuntimeException("Unable to analyze text: " + text, e);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/TextField.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/TextField.java Thu Jul 2 15:07:40 2015
@@ -146,16 +146,15 @@ public class TextField extends FieldType
source.reset();
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
- BytesRef bytes = termAtt.getBytesRef();
if (!source.incrementToken())
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned no terms for multiTerm term: " + part);
- termAtt.fillBytesRef();
+ BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef());
if (source.incrementToken())
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned too many terms for multiTerm term: " + part);
source.end();
- return BytesRef.deepCopyOf(bytes);
+ return bytes;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"error analyzing range part: " + part, e);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java?rev=1688845&r1=1688844&r2=1688845&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java Thu Jul 2 15:07:40 2015
@@ -267,10 +267,9 @@ public class CursorMarkTest extends Solr
String term = TestUtil.randomRealisticUnicodeString(random());
try (TokenStream ts = analyzer.tokenStream("fake", term)) {
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- val = termAtt.getBytesRef();
ts.reset();
assertTrue(ts.incrementToken());
- termAtt.fillBytesRef();
+ val = BytesRef.deepCopyOf(termAtt.getBytesRef());
assertFalse(ts.incrementToken());
ts.end();
}