You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2014/08/12 16:46:49 UTC
svn commit: r1617493 [1/8] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/java/org/apache/lucene/ana...
Author: jpountz
Date: Tue Aug 12 14:46:45 2014
New Revision: 1617493
URL: http://svn.apache.org/r1617493
Log:
LUCENE-5864: Split BytesRef into BytesRef and BytesRefBuilder.
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefBuilder.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/CharsRefBuilder.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IntsRefBuilder.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/WordnetSynonymParser.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
lucene/dev/trunk/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java
lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/CompressionTools.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AutomatonTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Terms.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/DataOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRef.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/CharsRef.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IntsRef.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/LongsRef.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Util.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueStr.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestBytesRef.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestBytesRefArray.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestBytesRefHash.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/mutable/TestMutableValues.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
lucene/dev/trunk/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
lucene/dev/trunk/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
lucene/dev/trunk/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
lucene/dev/trunk/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
lucene/dev/trunk/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java
lucene/dev/trunk/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java
lucene/dev/trunk/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
lucene/dev/trunk/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnumFrame.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JakartaRegexpCapabilities.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JavaUtilRegexCapabilities.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/valuesource/MultiStringFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/valuesource/SingleStringFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/EnumField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/CursorMark.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/QueryParsing.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/GroupConverter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DocBasedVersionConstraintsProcessorFactory.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Aug 12 14:46:45 2014
@@ -164,6 +164,10 @@ API Changes
* LUCENE-5850: CheckIndex now prints the Lucene version used to write
each segment. (Robert Muir, Mike McCandless)
+* LUCENE-5836: BytesRef has been splitted into BytesRef, whose intended usage is
+ to be just a reference to a section of a larger byte[] and BytesRefBuilder
+ which is a StringBuilder-like class for BytesRef instances. (Adrien Grand)
+
Optimizations
* LUCENE-5780: Make OrdinalMap more memory-efficient, especially in case the
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java Tue Aug 12 14:46:45 2014
@@ -24,6 +24,7 @@ import java.util.TreeMap;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
@@ -109,7 +110,7 @@ public class NormalizeCharMap {
try {
final Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
final org.apache.lucene.util.fst.Builder<CharsRef> builder = new org.apache.lucene.util.fst.Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
- final IntsRef scratch = new IntsRef();
+ final IntsRefBuilder scratch = new IntsRefBuilder();
for(Map.Entry<String,String> ent : pendingPairs.entrySet()) {
builder.add(Util.toUTF16(ent.getKey(), scratch),
new CharsRef(ent.getValue()));
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java Tue Aug 12 14:46:45 2014
@@ -20,10 +20,12 @@ package org.apache.lucene.analysis.hunsp
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
@@ -400,7 +402,7 @@ public class Dictionary {
private FST<IntsRef> affixFST(TreeMap<String,List<Integer>> affixes) throws IOException {
IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton();
Builder<IntsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, outputs);
- IntsRef scratch = new IntsRef();
+ IntsRefBuilder scratch = new IntsRefBuilder();
for (Map.Entry<String,List<Integer>> entry : affixes.entrySet()) {
Util.toUTF32(entry.getKey(), scratch);
List<Integer> entries = entry.getValue();
@@ -408,7 +410,7 @@ public class Dictionary {
for (Integer c : entries) {
output.ints[output.length++] = c;
}
- builder.add(scratch, output);
+ builder.add(scratch.get(), output);
}
return builder.finish();
}
@@ -450,7 +452,7 @@ public class Dictionary {
Map<String,Integer> seenPatterns,
Map<String,Integer> seenStrips) throws IOException, ParseException {
- BytesRef scratch = new BytesRef();
+ BytesRefBuilder scratch = new BytesRefBuilder();
StringBuilder sb = new StringBuilder();
String args[] = header.split("\\s+");
@@ -543,7 +545,7 @@ public class Dictionary {
}
encodeFlags(scratch, appendFlags);
- int appendFlagsOrd = flagLookup.add(scratch);
+ int appendFlagsOrd = flagLookup.add(scratch.get());
if (appendFlagsOrd < 0) {
// already exists in our hash
appendFlagsOrd = (-appendFlagsOrd)-1;
@@ -594,10 +596,10 @@ public class Dictionary {
Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
- IntsRef scratchInts = new IntsRef();
+ IntsRefBuilder scratchInts = new IntsRefBuilder();
for (Map.Entry<String,String> entry : mappings.entrySet()) {
Util.toUTF16(entry.getKey(), scratchInts);
- builder.add(scratchInts, new CharsRef(entry.getValue()));
+ builder.add(scratchInts.get(), new CharsRef(entry.getValue()));
}
return builder.finish();
@@ -768,8 +770,8 @@ public class Dictionary {
* @throws IOException Can be thrown while reading from the file
*/
private void readDictionaryFiles(List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException {
- BytesRef flagsScratch = new BytesRef();
- IntsRef scratchInts = new IntsRef();
+ BytesRefBuilder flagsScratch = new BytesRefBuilder();
+ IntsRefBuilder scratchInts = new IntsRefBuilder();
StringBuilder sb = new StringBuilder();
@@ -859,17 +861,17 @@ public class Dictionary {
unsorted.delete();
ByteSequencesReader reader = new ByteSequencesReader(sorted);
- BytesRef scratchLine = new BytesRef();
+ BytesRefBuilder scratchLine = new BytesRefBuilder();
// TODO: the flags themselves can be double-chars (long) or also numeric
// either way the trick is to encode them as char... but they must be parsed differently
String currentEntry = null;
- IntsRef currentOrds = new IntsRef();
+ IntsRefBuilder currentOrds = new IntsRefBuilder();
String line;
while (reader.read(scratchLine)) {
- line = scratchLine.utf8ToString();
+ line = scratchLine.get().utf8ToString();
String entry;
char wordForm[];
int end;
@@ -909,7 +911,7 @@ public class Dictionary {
throw new IllegalArgumentException("out of order: " + entry + " < " + currentEntry);
} else {
encodeFlags(flagsScratch, wordForm);
- int ord = flagLookup.add(flagsScratch);
+ int ord = flagLookup.add(flagsScratch.get());
if (ord < 0) {
// already exists in our hash
ord = (-ord)-1;
@@ -917,27 +919,25 @@ public class Dictionary {
// finalize current entry, and switch "current" if necessary
if (cmp > 0 && currentEntry != null) {
Util.toUTF32(currentEntry, scratchInts);
- words.add(scratchInts, currentOrds);
+ words.add(scratchInts.get(), currentOrds.get());
}
// swap current
if (cmp > 0 || currentEntry == null) {
currentEntry = entry;
- currentOrds = new IntsRef(); // must be this way
+ currentOrds = new IntsRefBuilder(); // must be this way
}
if (hasStemExceptions) {
- currentOrds.grow(currentOrds.length+2);
- currentOrds.ints[currentOrds.length++] = ord;
- currentOrds.ints[currentOrds.length++] = stemExceptionID;
+ currentOrds.append(ord);
+ currentOrds.append(stemExceptionID);
} else {
- currentOrds.grow(currentOrds.length+1);
- currentOrds.ints[currentOrds.length++] = ord;
+ currentOrds.append(ord);
}
}
}
// finalize last entry
Util.toUTF32(currentEntry, scratchInts);
- words.add(scratchInts, currentOrds);
+ words.add(scratchInts.get(), currentOrds.get());
reader.close();
sorted.delete();
@@ -957,15 +957,14 @@ public class Dictionary {
return flags;
}
- static void encodeFlags(BytesRef b, char flags[]) {
+ static void encodeFlags(BytesRefBuilder b, char flags[]) {
int len = flags.length << 1;
b.grow(len);
- b.length = len;
- int upto = b.offset;
+ b.clear();
for (int i = 0; i < flags.length; i++) {
int flag = flags[i];
- b.bytes[upto++] = (byte) ((flag >> 8) & 0xff);
- b.bytes[upto++] = (byte) (flag & 0xff);
+ b.append((byte) ((flag >> 8) & 0xff));
+ b.append((byte) (flag & 0xff));
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java Tue Aug 12 14:46:45 2014
@@ -17,23 +17,25 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
-import java.io.IOException;
-import java.util.ArrayList;
-
/**
* Provides the ability to override any {@link KeywordAttribute} aware stemmer
* with custom dictionary-based stemming.
@@ -45,7 +47,7 @@ public final class StemmerOverrideFilter
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final BytesReader fstReader;
private final Arc<BytesRef> scratchArc = new FST.Arc<>();
- private final CharsRef spare = new CharsRef();
+ private char[] spare = new char[0];
/**
* Create a new StemmerOverrideFilter, performing dictionary-based stemming
@@ -71,12 +73,13 @@ public final class StemmerOverrideFilter
if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms
final BytesRef stem = stemmerOverrideMap.get(termAtt.buffer(), termAtt.length(), scratchArc, fstReader);
if (stem != null) {
- final char[] buffer = spare.chars = termAtt.buffer();
- UnicodeUtil.UTF8toUTF16(stem.bytes, stem.offset, stem.length, spare);
- if (spare.chars != buffer) {
- termAtt.copyBuffer(spare.chars, spare.offset, spare.length);
+ spare = ArrayUtil.grow(termAtt.buffer(), stem.length);
+ final int length = UnicodeUtil.UTF8toUTF16(stem, spare);
+ if (spare != termAtt.buffer()) {
+ termAtt.copyBuffer(spare, 0, length);
+ } else {
+ termAtt.setLength(length);
}
- termAtt.setLength(spare.length);
keywordAtt.setKeyword(true);
}
}
@@ -144,10 +147,10 @@ public final class StemmerOverrideFilter
*/
public static class Builder {
private final BytesRefHash hash = new BytesRefHash();
- private final BytesRef spare = new BytesRef();
+ private final BytesRefBuilder spare = new BytesRefBuilder();
private final ArrayList<CharSequence> outputValues = new ArrayList<>();
private final boolean ignoreCase;
- private final CharsRef charsSpare = new CharsRef();
+ private final CharsRefBuilder charsSpare = new CharsRefBuilder();
/**
* Creates a new {@link Builder} with ignoreCase set to <code>false</code>
@@ -176,17 +179,17 @@ public final class StemmerOverrideFilter
if (ignoreCase) {
// convert on the fly to lowercase
charsSpare.grow(length);
- final char[] buffer = charsSpare.chars;
+ final char[] buffer = charsSpare.chars();
for (int i = 0; i < length; ) {
i += Character.toChars(
Character.toLowerCase(
Character.codePointAt(input, i)), buffer, i);
}
- UnicodeUtil.UTF16toUTF8(buffer, 0, length, spare);
+ spare.copyChars(buffer, 0, length);
} else {
- UnicodeUtil.UTF16toUTF8(input, 0, length, spare);
+ spare.copyChars(input, 0, length);
}
- if (hash.add(spare) >= 0) {
+ if (hash.add(spare.get()) >= 0) {
outputValues.add(output);
return true;
}
@@ -203,13 +206,14 @@ public final class StemmerOverrideFilter
org.apache.lucene.util.fst.Builder<BytesRef> builder = new org.apache.lucene.util.fst.Builder<>(
FST.INPUT_TYPE.BYTE4, outputs);
final int[] sort = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
- IntsRef intsSpare = new IntsRef();
+ IntsRefBuilder intsSpare = new IntsRefBuilder();
final int size = hash.size();
+ BytesRef spare = new BytesRef();
for (int i = 0; i < size; i++) {
int id = sort[i];
BytesRef bytesRef = hash.get(id, spare);
- UnicodeUtil.UTF8toUTF32(bytesRef, intsSpare);
- builder.add(intsSpare, new BytesRef(outputValues.get(id)));
+ intsSpare.copyUTF8Bytes(bytesRef);
+ builder.add(intsSpare.get(), new BytesRef(outputValues.get(id)));
}
return new StemmerOverrideMap(builder.finish(), ignoreCase);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Tue Aug 12 14:46:45 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.IOUtils;
import java.io.IOException;
@@ -123,11 +124,11 @@ public final class DutchAnalyzer extends
// we don't need to ignore case here since we lowercase in this analyzer anyway
StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(false);
CharArrayMap<String>.EntryIterator iter = stemOverrideDict.entrySet().iterator();
- CharsRef spare = new CharsRef();
+ CharsRefBuilder spare = new CharsRefBuilder();
while (iter.hasNext()) {
char[] nextKey = iter.nextKey();
spare.copyChars(nextKey, 0, nextKey.length);
- builder.add(spare, iter.currentValue());
+ builder.add(spare.get(), iter.currentValue());
}
try {
this.stemdict = builder.build();
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java Tue Aug 12 14:46:45 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
/**
* CaptureGroup uses Java regexes to emit multiple tokens - one for each capture
@@ -74,7 +75,7 @@ public final class PatternCaptureGroupTo
private final PositionIncrementAttribute posAttr = addAttribute(PositionIncrementAttribute.class);
private State state;
private final Matcher[] matchers;
- private final CharsRef spare = new CharsRef();
+ private final CharsRefBuilder spare = new CharsRefBuilder();
private final int[] groupCounts;
private final boolean preserveOriginal;
private int[] currentGroup;
@@ -119,7 +120,7 @@ public final class PatternCaptureGroupTo
final int start = matcher.start(currentGroup[i]);
final int end = matcher.end(currentGroup[i]);
if (start == end || preserveOriginal && start == 0
- && spare.length == end) {
+ && spare.length() == end) {
currentGroup[i]++;
continue;
}
@@ -151,7 +152,7 @@ public final class PatternCaptureGroupTo
.end(currentGroup[currentMatcher]);
posAttr.setPositionIncrement(0);
- charTermAttr.copyBuffer(spare.chars, start, end - start);
+ charTermAttr.copyBuffer(spare.chars(), start, end - start);
currentGroup[currentMatcher]++;
return true;
}
@@ -166,7 +167,7 @@ public final class PatternCaptureGroupTo
state = captureState();
for (int i = 0; i < matchers.length; i++) {
- matchers[i].reset(spare);
+ matchers[i].reset(spare.get());
currentGroup[i] = -1;
}
@@ -182,7 +183,7 @@ public final class PatternCaptureGroupTo
if (start == 0) {
charTermAttr.setLength(end);
} else {
- charTermAttr.copyBuffer(spare.chars, start, end - start);
+ charTermAttr.copyBuffer(spare.chars(), start, end - start);
}
currentGroup[currentMatcher]++;
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Tue Aug 12 14:46:45 2014
@@ -29,7 +29,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.UnicodeUtil;
/**
@@ -142,13 +142,13 @@ public final class QueryAutoStopWordAnal
for (String field : fields) {
Set<String> stopWords = new HashSet<>();
Terms terms = MultiFields.getTerms(indexReader, field);
- CharsRef spare = new CharsRef();
+ CharsRefBuilder spare = new CharsRefBuilder();
if (terms != null) {
TermsEnum te = terms.iterator(null);
BytesRef text;
while ((text = te.next()) != null) {
if (te.docFreq() > maxDocFreq) {
- UnicodeUtil.UTF8toUTF16(text, spare);
+ spare.copyUTF8Bytes(text);
stopWords.add(spare.toString());
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java Tue Aug 12 14:46:45 2014
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
/**
* Parser for the Solr synonyms format.
@@ -95,19 +96,19 @@ public class SolrSynonymParser extends S
String inputStrings[] = split(sides[0], ",");
inputs = new CharsRef[inputStrings.length];
for (int i = 0; i < inputs.length; i++) {
- inputs[i] = analyze(unescape(inputStrings[i]).trim(), new CharsRef());
+ inputs[i] = analyze(unescape(inputStrings[i]).trim(), new CharsRefBuilder());
}
String outputStrings[] = split(sides[1], ",");
outputs = new CharsRef[outputStrings.length];
for (int i = 0; i < outputs.length; i++) {
- outputs[i] = analyze(unescape(outputStrings[i]).trim(), new CharsRef());
+ outputs[i] = analyze(unescape(outputStrings[i]).trim(), new CharsRefBuilder());
}
} else {
String inputStrings[] = split(line, ",");
inputs = new CharsRef[inputStrings.length];
for (int i = 0; i < inputs.length; i++) {
- inputs[i] = analyze(unescape(inputStrings[i]).trim(), new CharsRef());
+ inputs[i] = analyze(unescape(inputStrings[i]).trim(), new CharsRefBuilder());
}
if (expand) {
outputs = inputs;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Tue Aug 12 14:46:45 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.synon
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -30,7 +31,9 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.FST;
@@ -134,7 +137,7 @@ public final class SynonymFilter extends
// state for (and enumerate) all other tokens at this
// position:
private static class PendingInput {
- final CharsRef term = new CharsRef();
+ final CharsRefBuilder term = new CharsRefBuilder();
AttributeSource.State state;
boolean keepOrig;
boolean matched;
@@ -157,7 +160,7 @@ public final class SynonymFilter extends
// Holds pending output synonyms for one future position:
private static class PendingOutputs {
- CharsRef[] outputs;
+ CharsRefBuilder[] outputs;
int[] endOffsets;
int[] posLengths;
int upto;
@@ -167,7 +170,7 @@ public final class SynonymFilter extends
int lastPosLength;
public PendingOutputs() {
- outputs = new CharsRef[1];
+ outputs = new CharsRefBuilder[1];
endOffsets = new int[1];
posLengths = new int[1];
}
@@ -181,12 +184,12 @@ public final class SynonymFilter extends
assert upto < count;
lastEndOffset = endOffsets[upto];
lastPosLength = posLengths[upto];
- final CharsRef result = outputs[upto++];
+ final CharsRefBuilder result = outputs[upto++];
posIncr = 0;
if (upto == count) {
reset();
}
- return result;
+ return result.get();
}
public int getLastEndOffset() {
@@ -199,9 +202,7 @@ public final class SynonymFilter extends
public void add(char[] output, int offset, int len, int endOffset, int posLength) {
if (count == outputs.length) {
- final CharsRef[] next = new CharsRef[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(outputs, 0, next, 0, count);
- outputs = next;
+ outputs = Arrays.copyOf(outputs, ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
}
if (count == endOffsets.length) {
final int[] next = new int[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_INT)];
@@ -214,7 +215,7 @@ public final class SynonymFilter extends
posLengths = next;
}
if (outputs[count] == null) {
- outputs[count] = new CharsRef();
+ outputs[count] = new CharsRefBuilder();
}
outputs[count].copyChars(output, offset, len);
// endOffset can be -1, in which case we should simply
@@ -249,7 +250,7 @@ public final class SynonymFilter extends
private final BytesRef scratchBytes = new BytesRef();
- private final CharsRef scratchChars = new CharsRef();
+ private final CharsRefBuilder scratchChars = new CharsRefBuilder();
/**
* @param input input tokenstream
@@ -378,8 +379,8 @@ public final class SynonymFilter extends
}
} else {
// Still in our lookahead
- buffer = futureInputs[curNextRead].term.chars;
- bufferLen = futureInputs[curNextRead].term.length;
+ buffer = futureInputs[curNextRead].term.chars();
+ bufferLen = futureInputs[curNextRead].term.length();
inputEndOffset = futureInputs[curNextRead].endOffset;
//System.out.println(" old token=" + new String(buffer, 0, bufferLen));
}
@@ -462,19 +463,19 @@ public final class SynonymFilter extends
synonyms.words.get(bytesReader.readVInt(),
scratchBytes);
//System.out.println(" outIDX=" + outputIDX + " bytes=" + scratchBytes.length);
- UnicodeUtil.UTF8toUTF16(scratchBytes, scratchChars);
- int lastStart = scratchChars.offset;
- final int chEnd = lastStart + scratchChars.length;
+ scratchChars.copyUTF8Bytes(scratchBytes);
+ int lastStart = 0;
+ final int chEnd = lastStart + scratchChars.length();
int outputUpto = nextRead;
for(int chIDX=lastStart;chIDX<=chEnd;chIDX++) {
- if (chIDX == chEnd || scratchChars.chars[chIDX] == SynonymMap.WORD_SEPARATOR) {
+ if (chIDX == chEnd || scratchChars.charAt(chIDX) == SynonymMap.WORD_SEPARATOR) {
final int outputLen = chIDX - lastStart;
// Caller is not allowed to have empty string in
// the output:
assert outputLen > 0: "output contains empty string: " + scratchChars;
final int endOffset;
final int posLen;
- if (chIDX == chEnd && lastStart == scratchChars.offset) {
+ if (chIDX == chEnd && lastStart == 0) {
// This rule had a single output token, so, we set
// this output's endOffset to the current
// endOffset (ie, endOffset of the last input
@@ -489,7 +490,7 @@ public final class SynonymFilter extends
endOffset = -1;
posLen = 1;
}
- futureOutputs[outputUpto].add(scratchChars.chars, lastStart, outputLen, endOffset, posLen);
+ futureOutputs[outputUpto].add(scratchChars.chars(), lastStart, outputLen, endOffset, posLen);
//System.out.println(" " + new String(scratchChars.chars, lastStart, outputLen) + " outputUpto=" + outputUpto);
lastStart = 1+chIDX;
//System.out.println(" slot=" + outputUpto + " keepOrig=" + keepOrig);
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Tue Aug 12 14:46:45 2014
@@ -32,9 +32,12 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
@@ -69,7 +72,7 @@ public class SynonymMap {
public static class Builder {
private final HashMap<CharsRef,MapEntry> workingSet = new HashMap<>();
private final BytesRefHash words = new BytesRefHash();
- private final BytesRef utf8Scratch = new BytesRef(8);
+ private final BytesRefBuilder utf8Scratch = new BytesRefBuilder();
private int maxHorizontalContext;
private final boolean dedup;
@@ -88,15 +91,15 @@ public class SynonymMap {
/** Sugar: just joins the provided terms with {@link
* SynonymMap#WORD_SEPARATOR}. reuse and its chars
* must not be null. */
- public static CharsRef join(String[] words, CharsRef reuse) {
+ public static CharsRef join(String[] words, CharsRefBuilder reuse) {
int upto = 0;
- char[] buffer = reuse.chars;
+ char[] buffer = reuse.chars();
for (String word : words) {
final int wordLen = word.length();
final int needed = (0 == upto ? wordLen : 1 + upto + wordLen); // Add 1 for WORD_SEPARATOR
if (needed > buffer.length) {
reuse.grow(needed);
- buffer = reuse.chars;
+ buffer = reuse.chars();
}
if (upto > 0) {
buffer[upto++] = SynonymMap.WORD_SEPARATOR;
@@ -105,8 +108,8 @@ public class SynonymMap {
word.getChars(0, wordLen, buffer, upto);
upto += wordLen;
}
- reuse.length = upto;
- return reuse;
+ reuse.setLength(upto);
+ return reuse.get();
}
@@ -153,9 +156,9 @@ public class SynonymMap {
assert !hasHoles(output): "output has holes: " + output;
//System.out.println("fmap.add input=" + input + " numInputWords=" + numInputWords + " output=" + output + " numOutputWords=" + numOutputWords);
- UnicodeUtil.UTF16toUTF8(output.chars, output.offset, output.length, utf8Scratch);
+ utf8Scratch.copyChars(output.chars, output.offset, output.length);
// lookup in hash
- int ord = words.add(utf8Scratch);
+ int ord = words.add(utf8Scratch.get());
if (ord < 0) {
// already exists in our hash
ord = (-ord)-1;
@@ -212,7 +215,7 @@ public class SynonymMap {
org.apache.lucene.util.fst.Builder<BytesRef> builder =
new org.apache.lucene.util.fst.Builder<>(FST.INPUT_TYPE.BYTE4, outputs);
- BytesRef scratch = new BytesRef(64);
+ BytesRefBuilder scratch = new BytesRefBuilder();
ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput();
final Set<Integer> dedupSet;
@@ -229,7 +232,7 @@ public class SynonymMap {
CharsRef sortedKeys[] = keys.toArray(new CharsRef[keys.size()]);
Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());
- final IntsRef scratchIntsRef = new IntsRef();
+ final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
//System.out.println("fmap.build");
for (int keyIdx = 0; keyIdx < sortedKeys.length; keyIdx++) {
@@ -241,8 +244,7 @@ public class SynonymMap {
int estimatedSize = 5 + numEntries * 5; // numEntries + one ord for each entry
scratch.grow(estimatedSize);
- scratchOutput.reset(scratch.bytes, scratch.offset, scratch.bytes.length);
- assert scratch.offset == 0;
+ scratchOutput.reset(scratch.bytes());
// now write our output data:
int count = 0;
@@ -265,17 +267,17 @@ public class SynonymMap {
final int vIntLen = pos2-pos;
// Move the count + includeOrig to the front of the byte[]:
- System.arraycopy(scratch.bytes, pos, spare, 0, vIntLen);
- System.arraycopy(scratch.bytes, 0, scratch.bytes, vIntLen, pos);
- System.arraycopy(spare, 0, scratch.bytes, 0, vIntLen);
+ System.arraycopy(scratch.bytes(), pos, spare, 0, vIntLen);
+ System.arraycopy(scratch.bytes(), 0, scratch.bytes(), vIntLen, pos);
+ System.arraycopy(spare, 0, scratch.bytes(), 0, vIntLen);
if (dedupSet != null) {
dedupSet.clear();
}
- scratch.length = scratchOutput.getPosition() - scratch.offset;
+ scratch.setLength(scratchOutput.getPosition());
//System.out.println(" add input=" + input + " output=" + scratch + " offset=" + scratch.offset + " length=" + scratch.length + " count=" + count);
- builder.add(Util.toUTF32(input, scratchIntsRef), BytesRef.deepCopyOf(scratch));
+ builder.add(Util.toUTF32(input, scratchIntsRef), scratch.toBytesRef());
}
FST<BytesRef> fst = builder.finish();
@@ -306,12 +308,12 @@ public class SynonymMap {
/** Sugar: analyzes the text with the analyzer and
* separates by {@link SynonymMap#WORD_SEPARATOR}.
* reuse and its chars must not be null. */
- public CharsRef analyze(String text, CharsRef reuse) throws IOException {
+ public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException {
try (TokenStream ts = analyzer.tokenStream("", text)) {
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
ts.reset();
- reuse.length = 0;
+ reuse.clear();
while (ts.incrementToken()) {
int length = termAtt.length();
if (length == 0) {
@@ -320,21 +322,21 @@ public class SynonymMap {
if (posIncAtt.getPositionIncrement() != 1) {
throw new IllegalArgumentException("term: " + text + " analyzed to a token with posinc != 1");
}
- reuse.grow(reuse.length + length + 1); /* current + word + separator */
- int end = reuse.offset + reuse.length;
- if (reuse.length > 0) {
- reuse.chars[end++] = SynonymMap.WORD_SEPARATOR;
- reuse.length++;
+ reuse.grow(reuse.length() + length + 1); /* current + word + separator */
+ int end = reuse.length();
+ if (reuse.length() > 0) {
+ reuse.setCharAt(end++, SynonymMap.WORD_SEPARATOR);
+ reuse.setLength(reuse.length() + 1);
}
- System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
- reuse.length += length;
+ System.arraycopy(termAtt.buffer(), 0, reuse.chars(), end, length);
+ reuse.setLength(reuse.length() + length);
}
ts.end();
}
- if (reuse.length == 0) {
+ if (reuse.length() == 0) {
throw new IllegalArgumentException("term: " + text + " was completely eliminated by analyzer");
}
- return reuse;
+ return reuse.get();
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/WordnetSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/WordnetSynonymParser.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/WordnetSynonymParser.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/WordnetSynonymParser.java Tue Aug 12 14:46:45 2014
@@ -21,9 +21,11 @@ import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Reader;
import java.text.ParseException;
+import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
/**
* Parser for wordnet prolog format
@@ -58,12 +60,10 @@ public class WordnetSynonymParser extend
}
if (synset.length <= synsetSize+1) {
- CharsRef larger[] = new CharsRef[synset.length * 2];
- System.arraycopy(synset, 0, larger, 0, synsetSize);
- synset = larger;
+ synset = Arrays.copyOf(synset, synset.length * 2);
}
- synset[synsetSize] = parseSynonym(line, synset[synsetSize]);
+ synset[synsetSize] = parseSynonym(line, new CharsRefBuilder());
synsetSize++;
lastSynSetID = synSetID;
}
@@ -79,9 +79,9 @@ public class WordnetSynonymParser extend
}
}
- private CharsRef parseSynonym(String line, CharsRef reuse) throws IOException {
+ private CharsRef parseSynonym(String line, CharsRefBuilder reuse) throws IOException {
if (reuse == null) {
- reuse = new CharsRef(8);
+ reuse = new CharsRefBuilder();
}
int start = line.indexOf('\'')+1;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java Tue Aug 12 14:46:45 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.CharSequenceOutputs;
@@ -187,23 +188,23 @@ public class TestDictionary extends Luce
public void testReplacements() throws Exception {
Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
- IntsRef scratchInts = new IntsRef();
+ IntsRefBuilder scratchInts = new IntsRefBuilder();
// a -> b
Util.toUTF16("a", scratchInts);
- builder.add(scratchInts, new CharsRef("b"));
+ builder.add(scratchInts.get(), new CharsRef("b"));
// ab -> c
Util.toUTF16("ab", scratchInts);
- builder.add(scratchInts, new CharsRef("c"));
+ builder.add(scratchInts.get(), new CharsRef("c"));
// c -> de
Util.toUTF16("c", scratchInts);
- builder.add(scratchInts, new CharsRef("de"));
+ builder.add(scratchInts.get(), new CharsRef("de"));
// def -> gh
Util.toUTF16("def", scratchInts);
- builder.add(scratchInts, new CharsRef("gh"));
+ builder.add(scratchInts.get(), new CharsRef("gh"));
FST<CharsRef> fst = builder.finish();
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java Tue Aug 12 14:46:45 2014
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.synonym.SynonymFilter;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.junit.Test;
import java.io.IOException;
@@ -68,11 +69,11 @@ public class TestLimitTokenPositionFilte
builder.add(new CharsRef("one"), new CharsRef("first"), true);
builder.add(new CharsRef("one"), new CharsRef("alpha"), true);
builder.add(new CharsRef("one"), new CharsRef("beguine"), true);
- CharsRef multiWordCharsRef = new CharsRef();
+ CharsRefBuilder multiWordCharsRef = new CharsRefBuilder();
SynonymMap.Builder.join(new String[]{"and", "indubitably", "single", "only"}, multiWordCharsRef);
- builder.add(new CharsRef("one"), multiWordCharsRef, true);
+ builder.add(new CharsRef("one"), multiWordCharsRef.get(), true);
SynonymMap.Builder.join(new String[]{"dopple", "ganger"}, multiWordCharsRef);
- builder.add(new CharsRef("two"), multiWordCharsRef, true);
+ builder.add(new CharsRef("two"), multiWordCharsRef.get(), true);
SynonymMap synonymMap = builder.build();
TokenStream stream = new SynonymFilter(tokenizer, synonymMap, true);
stream = new LimitTokenPositionFilter(stream, 3, consumeAll);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Tue Aug 12 14:46:45 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.analysis.MockGr
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.TestUtil;
public class TestSynonymMapFilter extends BaseTokenStreamTestCase {
@@ -54,13 +55,13 @@ public class TestSynonymMapFilter extend
if (VERBOSE) {
System.out.println(" add input=" + input + " output=" + output + " keepOrig=" + keepOrig);
}
- CharsRef inputCharsRef = new CharsRef();
+ CharsRefBuilder inputCharsRef = new CharsRefBuilder();
SynonymMap.Builder.join(input.split(" +"), inputCharsRef);
- CharsRef outputCharsRef = new CharsRef();
+ CharsRefBuilder outputCharsRef = new CharsRefBuilder();
SynonymMap.Builder.join(output.split(" +"), outputCharsRef);
- b.add(inputCharsRef, outputCharsRef, keepOrig);
+ b.add(inputCharsRef.get(), outputCharsRef.get(), keepOrig);
}
private void assertEquals(CharTermAttribute term, String expected) {
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java Tue Aug 12 14:46:45 2014
@@ -29,6 +29,7 @@ import java.util.TreeMap;
import org.apache.lucene.analysis.ja.util.CSVUtil;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
@@ -90,7 +91,7 @@ public final class UserDictionary implem
PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton();
Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE2, fstOutput);
- IntsRef scratch = new IntsRef();
+ IntsRefBuilder scratch = new IntsRefBuilder();
long ord = 0;
for (String[] values : featureEntries) {
@@ -114,11 +115,11 @@ public final class UserDictionary implem
// add mapping to FST
String token = values[0];
scratch.grow(token.length());
- scratch.length = token.length();
+ scratch.setLength(token.length());
for (int i = 0; i < token.length(); i++) {
- scratch.ints[i] = (int) token.charAt(i);
+ scratch.setIntAt(i, (int) token.charAt(i));
}
- fstBuilder.add(scratch, ord);
+ fstBuilder.add(scratch.get(), ord);
segmentations.add(wordIdAndLength);
ord++;
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java Tue Aug 12 14:46:45 2014
@@ -34,6 +34,7 @@ import java.util.List;
import org.apache.lucene.analysis.ja.util.DictionaryBuilder.DictionaryFormat;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
@@ -133,7 +134,7 @@ public class TokenInfoDictionaryBuilder
PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton();
Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE2, 0, 0, true, true, Integer.MAX_VALUE, fstOutput, true, PackedInts.DEFAULT, true, 15);
- IntsRef scratch = new IntsRef();
+ IntsRefBuilder scratch = new IntsRefBuilder();
long ord = -1; // first ord will be 0
String lastValue = null;
@@ -152,11 +153,11 @@ public class TokenInfoDictionaryBuilder
ord++;
lastValue = token;
scratch.grow(token.length());
- scratch.length = token.length();
+ scratch.setLength(token.length());
for (int i = 0; i < token.length(); i++) {
- scratch.ints[i] = (int) token.charAt(i);
+ scratch.setIntAt(i, (int) token.charAt(i));
}
- fstBuilder.add(scratch, ord);
+ fstBuilder.add(scratch.get(), ord);
}
dictionary.addMapping((int)ord, offset);
offset = next;
Modified: lucene/dev/trunk/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java Tue Aug 12 14:46:45 2014
@@ -48,7 +48,7 @@ public class MorfologikFilter extends To
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
- private final CharsRef scratch = new CharsRef(0);
+ private final CharsRefBuilder scratch = new CharsRefBuilder();
private final CharacterUtils charUtils = CharacterUtils.getInstance();
private State current;
@@ -151,16 +151,17 @@ public class MorfologikFilter extends To
* Convert to lowercase in-place.
*/
private CharSequence toLowercase(CharSequence chs) {
- final int length = scratch.length = chs.length();
+ final int length = chs.length();
+ scratch.setLength(length);
scratch.grow(length);
- char buffer[] = scratch.chars;
+ char buffer[] = scratch.chars();
for (int i = 0; i < length;) {
i += Character.toChars(
Character.toLowerCase(charUtils.codePointAt(chs, i)), buffer, i);
}
- return scratch;
+ return scratch.get();
}
/** Resets stems accumulator and hands over to superclass. */
Modified: lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java (original)
+++ lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java Tue Aug 12 14:46:45 2014
@@ -33,7 +33,9 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
@@ -234,11 +236,11 @@ public class BooleanPerceptronClassifier
private void updateFST(SortedMap<String,Double> weights) throws IOException {
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
- BytesRef scratchBytes = new BytesRef();
- IntsRef scratchInts = new IntsRef();
+ BytesRefBuilder scratchBytes = new BytesRefBuilder();
+ IntsRefBuilder scratchInts = new IntsRefBuilder();
for (Map.Entry<String,Double> entry : weights.entrySet()) {
scratchBytes.copyChars(entry.getKey());
- fstBuilder.add(Util.toIntsRef(scratchBytes, scratchInts), entry
+ fstBuilder.add(Util.toIntsRef(scratchBytes.get(), scratchInts), entry
.getValue().longValue());
}
fst = fstBuilder.finish();
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Tue Aug 12 14:46:45 2014
@@ -45,6 +45,7 @@ import org.apache.lucene.util.Accountabl
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.DoubleBarrelLRUCache;
import org.apache.lucene.util.RamUsageEstimator;
@@ -315,7 +316,7 @@ public class BlockTermsReader extends Fi
private final boolean doOrd;
private final FieldAndTerm fieldTerm = new FieldAndTerm();
private final TermsIndexReaderBase.FieldIndexEnum indexEnum;
- private final BytesRef term = new BytesRef();
+ private final BytesRefBuilder term = new BytesRefBuilder();
/* This is true if indexEnum is "still" seek'd to the index term
for the current term. We set it to true on seeking, and then it
@@ -396,7 +397,7 @@ public class BlockTermsReader extends Fi
// is after current term but before next index term:
if (indexIsCurrent) {
- final int cmp = BytesRef.getUTF8SortedAsUnicodeComparator().compare(term, target);
+ final int cmp = BytesRef.getUTF8SortedAsUnicodeComparator().compare(term.get(), target);
if (cmp == 0) {
// Already at the requested term
@@ -471,7 +472,7 @@ public class BlockTermsReader extends Fi
// First, see if target term matches common prefix
// in this block:
if (common < termBlockPrefix) {
- final int cmp = (term.bytes[common]&0xFF) - (target.bytes[target.offset + common]&0xFF);
+ final int cmp = (term.byteAt(common)&0xFF) - (target.bytes[target.offset + common]&0xFF);
if (cmp < 0) {
// TODO: maybe we should store common prefix
@@ -490,11 +491,9 @@ public class BlockTermsReader extends Fi
termSuffixesReader.skipBytes(termSuffixesReader.readVInt());
}
final int suffix = termSuffixesReader.readVInt();
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
}
state.ord++;
@@ -511,11 +510,9 @@ public class BlockTermsReader extends Fi
assert state.termBlockOrd == 0;
final int suffix = termSuffixesReader.readVInt();
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
return SeekStatus.NOT_FOUND;
} else {
common++;
@@ -548,22 +545,18 @@ public class BlockTermsReader extends Fi
} else if (cmp > 0) {
// Done! Current term is after target. Stop
// here, fill in real term, return NOT_FOUND.
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
//System.out.println(" NOT_FOUND");
return SeekStatus.NOT_FOUND;
}
}
if (!next && target.length <= termLen) {
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
if (target.length == termLen) {
// Done! Exact match. Stop here, fill in
@@ -578,11 +571,9 @@ public class BlockTermsReader extends Fi
if (state.termBlockOrd == blockTermCount) {
// Must pre-fill term for next block's common prefix
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
break;
} else {
termSuffixesReader.skipBytes(suffix);
@@ -653,23 +644,21 @@ public class BlockTermsReader extends Fi
final int suffix = termSuffixesReader.readVInt();
//System.out.println(" suffix=" + suffix);
- term.length = termBlockPrefix + suffix;
- if (term.bytes.length < term.length) {
- term.grow(term.length);
- }
- termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix);
+ term.setLength(termBlockPrefix + suffix);
+ term.grow(term.length());
+ termSuffixesReader.readBytes(term.bytes(), termBlockPrefix, suffix);
state.termBlockOrd++;
// NOTE: meaningless in the non-ord case
state.ord++;
//System.out.println(" return term=" + fieldInfo.name + ":" + term.utf8ToString() + " " + term + " tbOrd=" + state.termBlockOrd);
- return term;
+ return term.get();
}
@Override
public BytesRef term() {
- return term;
+ return term.get();
}
@Override
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java Tue Aug 12 14:46:45 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.blockte
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.lucene.codecs.BlockTermState;
@@ -39,6 +40,7 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
@@ -200,7 +202,7 @@ public class BlockTermsWriter extends Fi
}
private static class TermEntry {
- public final BytesRef term = new BytesRef();
+ public final BytesRefBuilder term = new BytesRefBuilder();
public BlockTermState state;
}
@@ -237,7 +239,7 @@ public class BlockTermsWriter extends Fi
this.longsSize = postingsWriter.setField(fieldInfo);
}
- private final BytesRef lastPrevTerm = new BytesRef();
+ private final BytesRefBuilder lastPrevTerm = new BytesRefBuilder();
void write(BytesRef text, TermsEnum termsEnum) throws IOException {
@@ -267,12 +269,10 @@ public class BlockTermsWriter extends Fi
}
if (pendingTerms.length == pendingCount) {
- final TermEntry[] newArray = new TermEntry[ArrayUtil.oversize(pendingCount+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(pendingTerms, 0, newArray, 0, pendingCount);
- for(int i=pendingCount;i<newArray.length;i++) {
- newArray[i] = new TermEntry();
+ pendingTerms = Arrays.copyOf(pendingTerms, ArrayUtil.oversize(pendingCount+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
+ for(int i=pendingCount;i<pendingTerms.length;i++) {
+ pendingTerms[i] = new TermEntry();
}
- pendingTerms = newArray;
}
final TermEntry te = pendingTerms[pendingCount];
te.term.copyBytes(text);
@@ -330,11 +330,11 @@ public class BlockTermsWriter extends Fi
// First pass: compute common prefix for all terms
// in the block, against term before first term in
// this block:
- int commonPrefix = sharedPrefix(lastPrevTerm, pendingTerms[0].term);
+ int commonPrefix = sharedPrefix(lastPrevTerm.get(), pendingTerms[0].term.get());
for(int termCount=1;termCount<pendingCount;termCount++) {
commonPrefix = Math.min(commonPrefix,
- sharedPrefix(lastPrevTerm,
- pendingTerms[termCount].term));
+ sharedPrefix(lastPrevTerm.get(),
+ pendingTerms[termCount].term.get()));
}
out.writeVInt(pendingCount);
@@ -342,11 +342,11 @@ public class BlockTermsWriter extends Fi
// 2nd pass: write suffixes, as separate byte[] blob
for(int termCount=0;termCount<pendingCount;termCount++) {
- final int suffix = pendingTerms[termCount].term.length - commonPrefix;
+ final int suffix = pendingTerms[termCount].term.length() - commonPrefix;
// TODO: cutover to better intblock codec, instead
// of interleaving here:
bytesWriter.writeVInt(suffix);
- bytesWriter.writeBytes(pendingTerms[termCount].term.bytes, commonPrefix, suffix);
+ bytesWriter.writeBytes(pendingTerms[termCount].term.bytes(), commonPrefix, suffix);
}
out.writeVInt((int) bytesWriter.getFilePointer());
bytesWriter.writeTo(out);
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java Tue Aug 12 14:46:45 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
@@ -131,7 +132,7 @@ public class FixedGapTermsIndexWriter ex
private RAMOutputStream addressBuffer = new RAMOutputStream();
private MonotonicBlockPackedWriter termAddresses = new MonotonicBlockPackedWriter(addressBuffer, BLOCKSIZE);
- private final BytesRef lastTerm = new BytesRef();
+ private final BytesRefBuilder lastTerm = new BytesRefBuilder();
SimpleFieldWriter(FieldInfo fieldInfo, long termsFilePointer) {
this.fieldInfo = fieldInfo;
@@ -163,7 +164,7 @@ public class FixedGapTermsIndexWriter ex
@Override
public void add(BytesRef text, TermStats stats, long termsFilePointer) throws IOException {
- final int indexedTermLength = indexedTermPrefixLength(lastTerm, text);
+ final int indexedTermLength = indexedTermPrefixLength(lastTerm.get(), text);
//System.out.println("FGW: add text=" + text.utf8ToString() + " " + text + " fp=" + termsFilePointer);
// write only the min prefix that shows the diff
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1617493&r1=1617492&r2=1617493&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java Tue Aug 12 14:46:45 2014
@@ -29,8 +29,10 @@ import org.apache.lucene.index.IndexFile
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
@@ -231,7 +233,7 @@ public class VariableGapTermsIndexWriter
FST<Long> fst;
final long indexStart;
- private final BytesRef lastTerm = new BytesRef();
+ private final BytesRefBuilder lastTerm = new BytesRefBuilder();
private boolean first = true;
public FSTFieldWriter(FieldInfo fieldInfo, long termsFilePointer) throws IOException {
@@ -261,7 +263,7 @@ public class VariableGapTermsIndexWriter
}
}
- private final IntsRef scratchIntsRef = new IntsRef();
+ private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
@Override
public void add(BytesRef text, TermStats stats, long termsFilePointer) throws IOException {
@@ -271,7 +273,7 @@ public class VariableGapTermsIndexWriter
return;
}
final int lengthSave = text.length;
- text.length = indexedTermPrefixLength(lastTerm, text);
+ text.length = indexedTermPrefixLength(lastTerm.get(), text);
try {
fstBuilder.add(Util.toIntsRef(text, scratchIntsRef), termsFilePointer);
} finally {