You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/01/03 20:46:53 UTC
[lucene-solr] branch master updated (beb163c -> 23e206d)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from beb163c In ContextQuery, use a more comprehensive check for an empty prefix automaton. (#383)
new 7f8b7ff LUCENE-9570: code reformatting [partial].
new 23e206d LUCENE-9570: code reformatting [record rev].
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.git-blame-ignore-revs | 3 +-
dev-tools/scripts/checkJavadocLinks.py | 4 +-
gradle/validation/spotless.gradle | 37 +-
.../analysis/icu/ICUCollationAttributeFactory.java | 82 +-
.../analysis/icu/ICUCollationDocValuesField.java | 31 +-
.../analysis/icu/ICUCollationKeyAnalyzer.java | 66 +-
.../lucene/analysis/icu/ICUFoldingFilter.java | 108 +-
.../analysis/icu/ICUFoldingFilterFactory.java | 10 +-
.../analysis/icu/ICUNormalizer2CharFilter.java | 33 +-
.../icu/ICUNormalizer2CharFilterFactory.java | 38 +-
.../lucene/analysis/icu/ICUNormalizer2Filter.java | 47 +-
.../analysis/icu/ICUNormalizer2FilterFactory.java | 39 +-
.../lucene/analysis/icu/ICUTransformFilter.java | 79 +-
.../analysis/icu/ICUTransformFilterFactory.java | 19 +-
.../apache/lucene/analysis/icu/package-info.java | 4 +-
.../icu/segmentation/BreakIteratorWrapper.java | 19 +-
.../icu/segmentation/CharArrayIterator.java | 12 +-
.../icu/segmentation/CompositeBreakIterator.java | 71 +-
.../segmentation/DefaultICUTokenizerConfig.java | 103 +-
.../analysis/icu/segmentation/ICUTokenizer.java | 105 +-
.../icu/segmentation/ICUTokenizerConfig.java | 12 +-
.../icu/segmentation/ICUTokenizerFactory.java | 91 +-
.../analysis/icu/segmentation/ScriptIterator.java | 86 +-
.../analysis/icu/segmentation/package-info.java | 4 +-
.../ICUCollatedTermAttributeImpl.java | 11 +-
.../icu/tokenattributes/ScriptAttribute.java | 15 +-
.../icu/tokenattributes/ScriptAttributeImpl.java | 26 +-
.../analysis/icu/tokenattributes/package-info.java | 4 +-
.../icu/TestICUCollationDocValuesField.java | 54 +-
.../analysis/icu/TestICUCollationKeyAnalyzer.java | 40 +-
.../lucene/analysis/icu/TestICUFoldingFilter.java | 86 +-
.../analysis/icu/TestICUFoldingFilterFactory.java | 28 +-
.../analysis/icu/TestICUNormalizer2CharFilter.java | 636 ++++++-----
.../icu/TestICUNormalizer2CharFilterFactory.java | 29 +-
.../analysis/icu/TestICUNormalizer2Filter.java | 94 +-
.../icu/TestICUNormalizer2FilterFactory.java | 31 +-
.../analysis/icu/TestICUTransformFilter.java | 98 +-
.../icu/TestICUTransformFilterFactory.java | 40 +-
.../icu/segmentation/TestCharArrayIterator.java | 32 +-
.../icu/segmentation/TestICUTokenizer.java | 552 ++++++---
.../icu/segmentation/TestICUTokenizerCJK.java | 104 +-
.../icu/segmentation/TestICUTokenizerFactory.java | 113 +-
.../icu/segmentation/TestMyanmarSyllable.java | 116 +-
.../icu/segmentation/TestWithCJKBigramFilter.java | 423 ++++---
.../analysis/icu/GenerateUTR30DataFiles.java | 81 +-
.../lucene/analysis/icu/RBBIRuleCompiler.java | 34 +-
.../lucene/analysis/ja/GraphvizFormatter.java | 93 +-
.../lucene/analysis/ja/JapaneseAnalyzer.java | 39 +-
.../lucene/analysis/ja/JapaneseBaseFormFilter.java | 15 +-
.../analysis/ja/JapaneseBaseFormFilterFactory.java | 10 +-
.../ja/JapaneseIterationMarkCharFilter.java | 105 +-
.../ja/JapaneseIterationMarkCharFilterFactory.java | 15 +-
.../analysis/ja/JapaneseKatakanaStemFilter.java | 36 +-
.../ja/JapaneseKatakanaStemFilterFactory.java | 18 +-
.../lucene/analysis/ja/JapaneseNumberFilter.java | 149 ++-
.../analysis/ja/JapaneseNumberFilterFactory.java | 15 +-
.../ja/JapanesePartOfSpeechStopFilter.java | 9 +-
.../ja/JapanesePartOfSpeechStopFilterFactory.java | 13 +-
.../analysis/ja/JapaneseReadingFormFilter.java | 14 +-
.../ja/JapaneseReadingFormFilterFactory.java | 12 +-
.../lucene/analysis/ja/JapaneseTokenizer.java | 736 +++++++-----
.../analysis/ja/JapaneseTokenizerFactory.java | 71 +-
.../java/org/apache/lucene/analysis/ja/Token.java | 90 +-
.../lucene/analysis/ja/dict/BinaryDictionary.java | 127 ++-
.../analysis/ja/dict/CharacterDefinition.java | 44 +-
.../lucene/analysis/ja/dict/ConnectionCosts.java | 22 +-
.../apache/lucene/analysis/ja/dict/Dictionary.java | 35 +-
.../analysis/ja/dict/TokenInfoDictionary.java | 27 +-
.../lucene/analysis/ja/dict/TokenInfoFST.java | 31 +-
.../lucene/analysis/ja/dict/UnknownDictionary.java | 32 +-
.../lucene/analysis/ja/dict/UserDictionary.java | 117 +-
.../lucene/analysis/ja/dict/package-info.java | 4 +-
.../apache/lucene/analysis/ja/package-info.java | 4 +-
.../ja/tokenattributes/BaseFormAttribute.java | 7 +-
.../ja/tokenattributes/BaseFormAttributeImpl.java | 11 +-
.../ja/tokenattributes/InflectionAttribute.java | 8 +-
.../tokenattributes/InflectionAttributeImpl.java | 16 +-
.../ja/tokenattributes/PartOfSpeechAttribute.java | 6 +-
.../tokenattributes/PartOfSpeechAttributeImpl.java | 17 +-
.../ja/tokenattributes/ReadingAttribute.java | 8 +-
.../ja/tokenattributes/ReadingAttributeImpl.java | 16 +-
.../analysis/ja/tokenattributes/package-info.java | 4 +-
.../analysis/ja/util/BinaryDictionaryWriter.java | 115 +-
.../apache/lucene/analysis/ja/util/CSVUtil.java | 58 +-
.../ja/util/CharacterDefinitionWriter.java | 37 +-
.../analysis/ja/util/ConnectionCostsBuilder.java | 6 +-
.../analysis/ja/util/ConnectionCostsWriter.java | 23 +-
.../lucene/analysis/ja/util/DictionaryBuilder.java | 42 +-
.../lucene/analysis/ja/util/ToStringUtil.java | 64 +-
.../ja/util/TokenInfoDictionaryBuilder.java | 46 +-
.../ja/util/TokenInfoDictionaryWriter.java | 10 +-
.../analysis/ja/util/UnknownDictionaryBuilder.java | 30 +-
.../analysis/ja/util/UnknownDictionaryWriter.java | 18 +-
.../lucene/analysis/ja/util/package-info.java | 4 +-
.../analysis/ja/StringMockResourceLoader.java | 2 -
.../org/apache/lucene/analysis/ja/TestCSVUtil.java | 9 +-
.../lucene/analysis/ja/TestExtendedMode.java | 39 +-
.../apache/lucene/analysis/ja/TestFactories.java | 73 +-
.../lucene/analysis/ja/TestJapaneseAnalyzer.java | 221 ++--
.../analysis/ja/TestJapaneseBaseFormFilter.java | 80 +-
.../ja/TestJapaneseBaseFormFilterFactory.java | 36 +-
.../ja/TestJapaneseIterationMarkCharFilter.java | 198 ++--
...TestJapaneseIterationMarkCharFilterFactory.java | 85 +-
.../ja/TestJapaneseKatakanaStemFilter.java | 97 +-
.../ja/TestJapaneseKatakanaStemFilterFactory.java | 63 +-
.../analysis/ja/TestJapaneseNumberFilter.java | 258 ++---
.../ja/TestJapaneseNumberFilterFactory.java | 31 +-
.../TestJapanesePartOfSpeechStopFilterFactory.java | 54 +-
.../analysis/ja/TestJapaneseReadingFormFilter.java | 137 ++-
.../ja/TestJapaneseReadingFormFilterFactory.java | 43 +-
.../lucene/analysis/ja/TestJapaneseTokenizer.java | 686 ++++++-----
.../analysis/ja/TestJapaneseTokenizerFactory.java | 147 ++-
.../apache/lucene/analysis/ja/TestSearchMode.java | 66 +-
.../analysis/ja/dict/TokenInfoDictionaryTest.java | 41 +-
.../analysis/ja/dict/UserDictionaryTest.java | 37 +-
.../lucene/analysis/ja/util/TestToStringUtil.java | 260 +++--
.../analysis/ja/util/UnknownDictionaryTest.java | 17 +-
.../analysis/morfologik/MorfologikAnalyzer.java | 30 +-
.../analysis/morfologik/MorfologikFilter.java | 66 +-
.../morfologik/MorfologikFilterFactory.java | 45 +-
.../morfologik/MorphosyntacticTagsAttribute.java | 20 +-
.../MorphosyntacticTagsAttributeImpl.java | 25 +-
.../lucene/analysis/morfologik/package-info.java | 15 +-
.../analysis/uk/UkrainianMorfologikAnalyzer.java | 60 +-
.../apache/lucene/analysis/uk/package-info.java | 4 +-
.../morfologik/TestMorfologikAnalyzer.java | 110 +-
.../morfologik/TestMorfologikFilterFactory.java | 40 +-
.../lucene/analysis/uk/TestUkrainianAnalyzer.java | 59 +-
.../apache/lucene/analysis/ko/DecompoundToken.java | 20 +-
.../apache/lucene/analysis/ko/DictionaryToken.java | 37 +-
.../lucene/analysis/ko/GraphvizFormatter.java | 95 +-
.../apache/lucene/analysis/ko/KoreanAnalyzer.java | 26 +-
.../lucene/analysis/ko/KoreanNumberFilter.java | 117 +-
.../analysis/ko/KoreanNumberFilterFactory.java | 13 +-
.../analysis/ko/KoreanPartOfSpeechStopFilter.java | 57 +-
.../ko/KoreanPartOfSpeechStopFilterFactory.java | 19 +-
.../analysis/ko/KoreanReadingFormFilter.java | 6 +-
.../ko/KoreanReadingFormFilterFactory.java | 10 +-
.../apache/lucene/analysis/ko/KoreanTokenizer.java | 394 ++++---
.../lucene/analysis/ko/KoreanTokenizerFactory.java | 39 +-
.../java/org/apache/lucene/analysis/ko/POS.java | 197 +---
.../java/org/apache/lucene/analysis/ko/Token.java | 48 +-
.../lucene/analysis/ko/dict/BinaryDictionary.java | 70 +-
.../analysis/ko/dict/CharacterDefinition.java | 35 +-
.../lucene/analysis/ko/dict/ConnectionCosts.java | 20 +-
.../apache/lucene/analysis/ko/dict/Dictionary.java | 41 +-
.../analysis/ko/dict/TokenInfoDictionary.java | 24 +-
.../lucene/analysis/ko/dict/TokenInfoFST.java | 21 +-
.../lucene/analysis/ko/dict/UnknownDictionary.java | 7 +-
.../lucene/analysis/ko/dict/UserDictionary.java | 32 +-
.../lucene/analysis/ko/dict/package-info.java | 4 +-
.../apache/lucene/analysis/ko/package-info.java | 4 +-
.../ko/tokenattributes/PartOfSpeechAttribute.java | 21 +-
.../tokenattributes/PartOfSpeechAttributeImpl.java | 22 +-
.../ko/tokenattributes/ReadingAttribute.java | 13 +-
.../ko/tokenattributes/ReadingAttributeImpl.java | 5 +-
.../analysis/ko/tokenattributes/package-info.java | 4 +-
.../analysis/ko/util/BinaryDictionaryWriter.java | 49 +-
.../apache/lucene/analysis/ko/util/CSVUtil.java | 46 +-
.../ko/util/CharacterDefinitionWriter.java | 36 +-
.../analysis/ko/util/ConnectionCostsBuilder.java | 5 +-
.../analysis/ko/util/ConnectionCostsWriter.java | 22 +-
.../lucene/analysis/ko/util/DictionaryBuilder.java | 24 +-
.../ko/util/TokenInfoDictionaryBuilder.java | 31 +-
.../ko/util/TokenInfoDictionaryWriter.java | 9 +-
.../analysis/ko/util/UnknownDictionaryBuilder.java | 29 +-
.../analysis/ko/util/UnknownDictionaryWriter.java | 15 +-
.../lucene/analysis/ko/util/package-info.java | 4 +-
.../analysis/ko/StringMockResourceLoader.java | 2 -
.../lucene/analysis/ko/TestKoreanAnalyzer.java | 102 +-
.../lucene/analysis/ko/TestKoreanNumberFilter.java | 272 ++---
.../analysis/ko/TestKoreanNumberFilterFactory.java | 28 +-
.../TestKoreanPartOfSpeechStopFilterFactory.java | 31 +-
.../analysis/ko/TestKoreanReadingFormFilter.java | 50 +-
.../ko/TestKoreanReadingFormFilterFactory.java | 29 +-
.../lucene/analysis/ko/TestKoreanTokenizer.java | 814 +++++++------
.../analysis/ko/TestKoreanTokenizerFactory.java | 85 +-
.../analysis/ko/dict/TokenInfoDictionaryTest.java | 39 +-
.../analysis/ko/dict/UserDictionaryTest.java | 6 +-
.../analysis/ko/util/UnknownDictionaryTest.java | 11 +-
.../analysis/opennlp/OpenNLPChunkerFilter.java | 15 +-
.../opennlp/OpenNLPChunkerFilterFactory.java | 6 +-
.../analysis/opennlp/OpenNLPLemmatizerFilter.java | 38 +-
.../opennlp/OpenNLPLemmatizerFilterFactory.java | 20 +-
.../lucene/analysis/opennlp/OpenNLPPOSFilter.java | 12 +-
.../analysis/opennlp/OpenNLPPOSFilterFactory.java | 6 +-
.../opennlp/OpenNLPSentenceBreakIterator.java | 19 +-
.../lucene/analysis/opennlp/OpenNLPTokenizer.java | 28 +-
.../analysis/opennlp/OpenNLPTokenizerFactory.java | 10 +-
.../lucene/analysis/opennlp/package-info.java | 4 +-
.../analysis/opennlp/tools/NLPChunkerOp.java | 8 +-
.../analysis/opennlp/tools/NLPLemmatizerOp.java | 39 +-
.../analysis/opennlp/tools/NLPNERTaggerOp.java | 24 +-
.../analysis/opennlp/tools/NLPPOSTaggerOp.java | 6 +-
.../opennlp/tools/NLPSentenceDetectorOp.java | 6 +-
.../analysis/opennlp/tools/NLPTokenizerOp.java | 4 +-
.../analysis/opennlp/tools/OpenNLPOpsFactory.java | 53 +-
.../analysis/opennlp/tools/package-info.java | 4 +-
.../opennlp/TestOpenNLPChunkerFilterFactory.java | 99 +-
.../TestOpenNLPLemmatizerFilterFactory.java | 332 ++++--
.../opennlp/TestOpenNLPPOSFilterFactory.java | 129 ++-
.../opennlp/TestOpenNLPSentenceBreakIterator.java | 71 +-
.../opennlp/TestOpenNLPTokenizerFactory.java | 95 +-
.../analysis/phonetic/BeiderMorseFilter.java | 34 +-
.../phonetic/BeiderMorseFilterFactory.java | 23 +-
.../phonetic/DaitchMokotoffSoundexFilter.java | 5 +-
.../DaitchMokotoffSoundexFilterFactory.java | 19 +-
.../analysis/phonetic/DoubleMetaphoneFilter.java | 36 +-
.../phonetic/DoubleMetaphoneFilterFactory.java | 14 +-
.../lucene/analysis/phonetic/PhoneticFilter.java | 32 +-
.../analysis/phonetic/PhoneticFilterFactory.java | 64 +-
.../lucene/analysis/phonetic/package-info.java | 4 +-
.../phonetic/DoubleMetaphoneFilterTest.java | 73 +-
.../analysis/phonetic/TestBeiderMorseFilter.java | 167 +--
.../phonetic/TestBeiderMorseFilterFactory.java | 62 +-
.../phonetic/TestDaitchMokotoffSoundexFilter.java | 73 +-
.../TestDaitchMokotoffSoundexFilterFactory.java | 29 +-
.../phonetic/TestDoubleMetaphoneFilterFactory.java | 29 +-
.../analysis/phonetic/TestPhoneticFilter.java | 185 +--
.../phonetic/TestPhoneticFilterFactory.java | 230 ++--
.../lucene/analysis/cn/smart/AnalyzerProfile.java | 35 +-
.../apache/lucene/analysis/cn/smart/CharType.java | 56 +-
.../analysis/cn/smart/HMMChineseTokenizer.java | 19 +-
.../cn/smart/HMMChineseTokenizerFactory.java | 15 +-
.../analysis/cn/smart/SmartChineseAnalyzer.java | 82 +-
.../apache/lucene/analysis/cn/smart/Utility.java | 103 +-
.../lucene/analysis/cn/smart/WordSegmenter.java | 27 +-
.../apache/lucene/analysis/cn/smart/WordType.java | 50 +-
.../analysis/cn/smart/hhmm/AbstractDictionary.java | 66 +-
.../lucene/analysis/cn/smart/hhmm/BiSegGraph.java | 52 +-
.../analysis/cn/smart/hhmm/BigramDictionary.java | 59 +-
.../analysis/cn/smart/hhmm/HHMMSegmenter.java | 45 +-
.../lucene/analysis/cn/smart/hhmm/PathNode.java | 39 +-
.../lucene/analysis/cn/smart/hhmm/SegGraph.java | 30 +-
.../lucene/analysis/cn/smart/hhmm/SegToken.java | 66 +-
.../analysis/cn/smart/hhmm/SegTokenFilter.java | 15 +-
.../analysis/cn/smart/hhmm/SegTokenPair.java | 42 +-
.../analysis/cn/smart/hhmm/WordDictionary.java | 196 ++--
.../analysis/cn/smart/hhmm/package-info.java | 1 +
.../lucene/analysis/cn/smart/package-info.java | 33 +-
.../cn/smart/TestHMMChineseTokenizerFactory.java | 31 +-
.../cn/smart/TestSmartChineseAnalyzer.java | 121 +-
.../apache/lucene/analysis/pl/PolishAnalyzer.java | 76 +-
.../apache/lucene/analysis/pl/package-info.java | 4 +-
.../lucene/analysis/stempel/StempelFilter.java | 32 +-
.../stempel/StempelPolishStemFilterFactory.java | 9 +-
.../lucene/analysis/stempel/StempelStemmer.java | 45 +-
.../lucene/analysis/stempel/package-info.java | 4 +-
.../stempel/src/java/org/egothor/stemmer/Cell.java | 110 +-
.../src/java/org/egothor/stemmer/Compile.java | 159 ++-
.../stempel/src/java/org/egothor/stemmer/Diff.java | 158 ++-
.../src/java/org/egothor/stemmer/DiffIt.java | 125 +-
.../src/java/org/egothor/stemmer/Gener.java | 127 +--
.../stempel/src/java/org/egothor/stemmer/Lift.java | 135 ++-
.../src/java/org/egothor/stemmer/MultiTrie.java | 172 ++-
.../src/java/org/egothor/stemmer/MultiTrie2.java | 179 ++-
.../src/java/org/egothor/stemmer/Optimizer.java | 156 ++-
.../src/java/org/egothor/stemmer/Optimizer2.java | 118 +-
.../src/java/org/egothor/stemmer/Reduce.java | 133 ++-
.../stempel/src/java/org/egothor/stemmer/Row.java | 200 ++--
.../stempel/src/java/org/egothor/stemmer/Trie.java | 252 ++---
.../src/java/org/egothor/stemmer/package-info.java | 4 +-
.../lucene/analysis/pl/TestPolishAnalyzer.java | 14 +-
.../TestStempelPolishStemFilterFactory.java | 30 +-
.../src/test/org/egothor/stemmer/TestCompile.java | 127 +--
.../src/test/org/egothor/stemmer/TestStemmer.java | 189 ++--
.../lucene/backward_codecs/lucene50/ForUtil.java | 92 +-
.../lucene50/Lucene50PostingsFormat.java | 626 +++++-----
.../lucene50/Lucene50PostingsReader.java | 454 +++++---
.../lucene50/Lucene50ScoreSkipReader.java | 60 +-
.../lucene50/Lucene50SkipReader.java | 69 +-
.../lucene50/Lucene50StoredFieldsFormat.java | 139 +--
.../backward_codecs/lucene50/package-info.java | 2 +-
.../lucene60/Lucene60PointsFormat.java | 67 +-
.../lucene60/Lucene60PointsReader.java | 64 +-
.../backward_codecs/lucene60/package-info.java | 2 +-
.../lucene70/Lucene70SegmentInfoFormat.java | 126 ++-
.../backward_codecs/lucene70/package-info.java | 4 +-
.../backward_codecs/lucene80/Lucene80Codec.java | 56 +-
.../backward_codecs/lucene80/package-info.java | 2 +-
.../backward_codecs/lucene84/Lucene84Codec.java | 85 +-
.../backward_codecs/lucene84/package-info.java | 2 +-
.../backward_codecs/lucene86/Lucene86Codec.java | 81 +-
.../backward_codecs/lucene86/package-info.java | 2 +-
.../backward_codecs/lucene87/Lucene87Codec.java | 78 +-
.../backward_codecs/lucene87/package-info.java | 2 +-
.../lucene/backward_codecs/package-info.java | 2 +-
.../lucene50/Lucene50PostingsWriter.java | 121 +-
.../lucene50/Lucene50RWPostingsFormat.java | 12 +-
.../lucene50/Lucene50RWStoredFieldsFormat.java | 20 +-
.../lucene50/Lucene50SkipWriter.java | 68 +-
.../lucene50/TestBlockPostingsFormat.java | 22 +-
.../lucene50/TestBlockPostingsFormat2.java | 30 +-
.../lucene50/TestBlockPostingsFormat3.java | 253 +++--
.../backward_codecs/lucene50/TestForUtil.java | 25 +-
.../lucene50/TestLucene50StoredFieldsFormat.java | 1 -
...tLucene50StoredFieldsFormatHighCompression.java | 28 +-
...estLucene50StoredFieldsFormatMergeInstance.java | 5 +-
.../lucene60/Lucene60PointsWriter.java | 163 +--
.../lucene60/Lucene60RWPointsFormat.java | 2 -
.../lucene60/TestLucene60PointsFormat.java | 212 ++--
.../lucene70/Lucene70RWSegmentInfoFormat.java | 46 +-
.../lucene70/TestLucene70SegmentInfoFormat.java | 4 +-
.../backward_codecs/lucene84/Lucene84RWCodec.java | 7 +-
.../backward_codecs/lucene86/Lucene86RWCodec.java | 7 +-
.../backward_index/TestBackwardsCompatibility.java | 1063 +++++++++--------
.../backward_index/TestIndexWriterOnOldIndex.java | 18 +-
.../backward_index/TestManyPointsInOldIndex.java | 29 +-
.../org/apache/lucene/benchmark/Constants.java | 16 +-
.../apache/lucene/benchmark/byTask/Benchmark.java | 68 +-
.../lucene/benchmark/byTask/PerfRunData.java | 243 ++--
.../benchmark/byTask/feeds/AbstractQueryMaker.java | 24 +-
.../benchmark/byTask/feeds/ContentItemsSource.java | 147 ++-
.../benchmark/byTask/feeds/ContentSource.java | 24 +-
.../benchmark/byTask/feeds/DemoHTMLParser.java | 212 ++--
.../benchmark/byTask/feeds/DirContentSource.java | 63 +-
.../lucene/benchmark/byTask/feeds/DocData.java | 16 +-
.../lucene/benchmark/byTask/feeds/DocMaker.java | 176 ++-
.../byTask/feeds/EnwikiContentSource.java | 95 +-
.../benchmark/byTask/feeds/EnwikiQueryMaker.java | 157 ++-
.../lucene/benchmark/byTask/feeds/FacetSource.java | 15 +-
.../byTask/feeds/FileBasedQueryMaker.java | 57 +-
.../benchmark/byTask/feeds/GeonamesLineParser.java | 13 +-
.../lucene/benchmark/byTask/feeds/HTMLParser.java | 19 +-
.../benchmark/byTask/feeds/LineDocSource.java | 160 +--
.../byTask/feeds/LongToEnglishContentSource.java | 28 +-
.../byTask/feeds/LongToEnglishQueryMaker.java | 17 +-
.../byTask/feeds/NoMoreDataException.java | 11 +-
.../lucene/benchmark/byTask/feeds/QueryMaker.java | 27 +-
.../benchmark/byTask/feeds/RandomFacetSource.java | 38 +-
.../byTask/feeds/ReutersContentSource.java | 34 +-
.../benchmark/byTask/feeds/ReutersQueryMaker.java | 97 +-
.../benchmark/byTask/feeds/SimpleQueryMaker.java | 36 +-
.../byTask/feeds/SimpleSloppyPhraseQueryMaker.java | 21 +-
.../benchmark/byTask/feeds/SingleDocSource.java | 42 +-
.../byTask/feeds/SortableSingleDocSource.java | 272 ++++-
.../benchmark/byTask/feeds/SpatialDocMaker.java | 155 +--
.../byTask/feeds/SpatialFileQueryMaker.java | 32 +-
.../benchmark/byTask/feeds/TrecContentSource.java | 134 ++-
.../benchmark/byTask/feeds/TrecDocParser.java | 110 +-
.../benchmark/byTask/feeds/TrecFBISParser.java | 25 +-
.../benchmark/byTask/feeds/TrecFR94Parser.java | 34 +-
.../benchmark/byTask/feeds/TrecFTParser.java | 19 +-
.../benchmark/byTask/feeds/TrecGov2Parser.java | 17 +-
.../benchmark/byTask/feeds/TrecLATimesParser.java | 33 +-
.../benchmark/byTask/feeds/TrecParserByPath.java | 16 +-
.../benchmark/byTask/feeds/package-info.java | 4 +-
.../lucene/benchmark/byTask/package-info.java | 972 +++++++---------
.../benchmark/byTask/programmatic/Sample.java | 53 +-
.../byTask/programmatic/package-info.java | 4 +-
.../lucene/benchmark/byTask/stats/Points.java | 46 +-
.../lucene/benchmark/byTask/stats/Report.java | 23 +-
.../lucene/benchmark/byTask/stats/TaskStats.java | 99 +-
.../benchmark/byTask/stats/package-info.java | 4 +-
.../lucene/benchmark/byTask/tasks/AddDocTask.java | 24 +-
.../benchmark/byTask/tasks/AddFacetedDocTask.java | 35 +-
.../benchmark/byTask/tasks/AddIndexesTask.java | 41 +-
.../byTask/tasks/AnalyzerFactoryTask.java | 531 +++++----
.../benchmark/byTask/tasks/ClearStatsTask.java | 6 +-
.../benchmark/byTask/tasks/CloseIndexTask.java | 8 +-
.../benchmark/byTask/tasks/CloseReaderTask.java | 12 +-
.../byTask/tasks/CloseTaxonomyIndexTask.java | 7 +-
.../byTask/tasks/CloseTaxonomyReaderTask.java | 11 +-
.../benchmark/byTask/tasks/CommitIndexTask.java | 17 +-
.../byTask/tasks/CommitTaxonomyIndexTask.java | 8 +-
.../byTask/tasks/ConsumeContentSourceTask.java | 6 +-
.../benchmark/byTask/tasks/CreateIndexTask.java | 117 +-
.../byTask/tasks/CreateTaxonomyIndexTask.java | 13 +-
.../benchmark/byTask/tasks/ForceMergeTask.java | 9 +-
.../byTask/tasks/NearRealtimeReaderTask.java | 30 +-
.../benchmark/byTask/tasks/NewAnalyzerTask.java | 69 +-
.../byTask/tasks/NewCollationAnalyzerTask.java | 64 +-
.../benchmark/byTask/tasks/NewLocaleTask.java | 54 +-
.../benchmark/byTask/tasks/NewRoundTask.java | 6 +-
.../benchmark/byTask/tasks/OpenIndexTask.java | 27 +-
.../benchmark/byTask/tasks/OpenReaderTask.java | 16 +-
.../byTask/tasks/OpenTaxonomyIndexTask.java | 10 +-
.../byTask/tasks/OpenTaxonomyReaderTask.java | 7 +-
.../lucene/benchmark/byTask/tasks/PerfTask.java | 164 ++-
.../benchmark/byTask/tasks/PrintReaderTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/ReadTask.java | 83 +-
.../benchmark/byTask/tasks/ReadTokensTask.java | 33 +-
.../benchmark/byTask/tasks/ReopenReaderTask.java | 7 +-
.../lucene/benchmark/byTask/tasks/RepAllTask.java | 27 +-
.../byTask/tasks/RepSelectByPrefTask.java | 36 +-
.../byTask/tasks/RepSumByNameRoundTask.java | 30 +-
.../benchmark/byTask/tasks/RepSumByNameTask.java | 29 +-
.../byTask/tasks/RepSumByPrefRoundTask.java | 33 +-
.../benchmark/byTask/tasks/RepSumByPrefTask.java | 39 +-
.../lucene/benchmark/byTask/tasks/ReportTask.java | 86 +-
.../benchmark/byTask/tasks/ResetInputsTask.java | 11 +-
.../byTask/tasks/ResetSystemEraseTask.java | 13 +-
.../byTask/tasks/ResetSystemSoftTask.java | 11 +-
.../benchmark/byTask/tasks/RollbackIndexTask.java | 5 +-
.../lucene/benchmark/byTask/tasks/SearchTask.java | 9 +-
.../byTask/tasks/SearchTravRetHighlightTask.java | 147 ++-
.../tasks/SearchTravRetLoadFieldSelectorTask.java | 27 +-
.../benchmark/byTask/tasks/SearchTravRetTask.java | 19 +-
.../benchmark/byTask/tasks/SearchTravTask.java | 19 +-
.../byTask/tasks/SearchWithCollectorTask.java | 11 +-
.../benchmark/byTask/tasks/SearchWithSortTask.java | 26 +-
.../lucene/benchmark/byTask/tasks/SetPropTask.java | 28 +-
.../benchmark/byTask/tasks/TaskSequence.java | 159 ++-
.../benchmark/byTask/tasks/UpdateDocTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/WaitTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/WarmTask.java | 15 +-
.../byTask/tasks/WriteEnwikiLineDocTask.java | 22 +-
.../benchmark/byTask/tasks/WriteLineDocTask.java | 129 +--
.../benchmark/byTask/tasks/package-info.java | 4 +-
.../lucene/benchmark/byTask/utils/Algorithm.java | 245 ++--
.../benchmark/byTask/utils/AnalyzerFactory.java | 33 +-
.../lucene/benchmark/byTask/utils/Config.java | 97 +-
.../lucene/benchmark/byTask/utils/Format.java | 17 +-
.../lucene/benchmark/byTask/utils/StreamUtils.java | 37 +-
.../benchmark/byTask/utils/package-info.java | 4 +-
.../org/apache/lucene/benchmark/package-info.java | 51 +-
.../org/apache/lucene/benchmark/quality/Judge.java | 29 +-
.../lucene/benchmark/quality/QualityBenchmark.java | 127 ++-
.../lucene/benchmark/quality/QualityQuery.java | 43 +-
.../benchmark/quality/QualityQueryParser.java | 6 +-
.../lucene/benchmark/quality/QualityStats.java | 183 ++-
.../lucene/benchmark/quality/package-info.java | 66 +-
.../lucene/benchmark/quality/trec/QueryDriver.java | 45 +-
.../benchmark/quality/trec/Trec1MQReader.java | 36 +-
.../lucene/benchmark/quality/trec/TrecJudge.java | 71 +-
.../benchmark/quality/trec/TrecTopicsReader.java | 64 +-
.../benchmark/quality/trec/package-info.java | 4 +-
.../benchmark/quality/utils/DocNameExtractor.java | 52 +-
.../quality/utils/QualityQueriesFinder.java | 92 +-
.../benchmark/quality/utils/SimpleQQParser.java | 19 +-
.../benchmark/quality/utils/SubmissionReport.java | 59 +-
.../benchmark/quality/utils/package-info.java | 4 +-
.../lucene/benchmark/utils/ExtractReuters.java | 32 +-
.../lucene/benchmark/utils/ExtractWikipedia.java | 32 +-
.../lucene/benchmark/utils/package-info.java | 4 +-
.../apache/lucene/benchmark/BenchmarkTestCase.java | 36 +-
.../benchmark/byTask/TestPerfTasksLogic.java | 949 ++++++++--------
.../benchmark/byTask/TestPerfTasksParse.java | 92 +-
.../benchmark/byTask/feeds/DocMakerTest.java | 61 +-
.../byTask/feeds/EnwikiContentSourceTest.java | 144 ++-
.../benchmark/byTask/feeds/LineDocSourceTest.java | 101 +-
.../benchmark/byTask/feeds/TestHtmlParser.java | 93 +-
.../byTask/feeds/TrecContentSourceTest.java | 509 +++++----
.../benchmark/byTask/tasks/AddIndexesTaskTest.java | 47 +-
.../byTask/tasks/CommitIndexTaskTest.java | 5 +-
.../byTask/tasks/CountingSearchTestTask.java | 6 +-
.../byTask/tasks/CreateIndexTaskTest.java | 12 +-
.../benchmark/byTask/tasks/PerfTaskTest.java | 28 +-
.../byTask/tasks/SearchWithSortTaskTest.java | 4 +-
.../byTask/tasks/WriteEnwikiLineDocTaskTest.java | 37 +-
.../byTask/tasks/WriteLineDocTaskTest.java | 106 +-
.../byTask/tasks/alt/AltPackageTaskTest.java | 18 +-
.../benchmark/byTask/tasks/alt/AltTestTask.java | 8 +-
.../benchmark/byTask/utils/StreamUtilsTest.java | 32 +-
.../lucene/benchmark/byTask/utils/TestConfig.java | 3 -
.../lucene/benchmark/quality/TestQualityRun.java | 222 ++--
.../lucene/classification/BM25NBClassifier.java | 64 +-
.../BooleanPerceptronClassifier.java | 104 +-
.../CachingNaiveBayesClassifier.java | 73 +-
.../classification/ClassificationResult.java | 5 +-
.../apache/lucene/classification/Classifier.java | 16 +-
.../classification/KNearestFuzzyClassifier.java | 90 +-
.../classification/KNearestNeighborClassifier.java | 139 +--
.../classification/SimpleNaiveBayesClassifier.java | 90 +-
.../document/DocumentClassifier.java | 36 +-
.../KNearestNeighborDocumentClassifier.java | 72 +-
.../SimpleNaiveBayesDocumentClassifier.java | 84 +-
.../classification/document/package-info.java | 3 +-
.../apache/lucene/classification/package-info.java | 7 +-
.../utils/ConfusionMatrixGenerator.java | 89 +-
.../classification/utils/DatasetSplitter.java | 47 +-
.../utils/DocToDoubleVectorUtils.java | 25 +-
.../classification/utils/NearestFuzzyQuery.java | 101 +-
.../lucene/classification/utils/package-info.java | 4 +-
.../classification/BM25NBClassifierTest.java | 32 +-
.../BooleanPerceptronClassifierTest.java | 26 +-
.../CachingNaiveBayesClassifierTest.java | 48 +-
.../classification/ClassificationTestBase.java | 142 ++-
.../KNearestFuzzyClassifierTest.java | 22 +-
.../KNearestNeighborClassifierTest.java | 103 +-
.../SimpleNaiveBayesClassifierTest.java | 33 +-
.../Test20NewsgroupsClassification.java | 256 ++++-
.../document/DocumentClassificationTestBase.java | 150 +--
.../KNearestNeighborDocumentClassifierTest.java | 164 ++-
.../SimpleNaiveBayesDocumentClassifierTest.java | 116 +-
.../utils/ConfusionMatrixGeneratorTest.java | 97 +-
.../classification/utils/DataSplitterTest.java | 28 +-
.../utils/DocToDoubleVectorUtilsTest.java | 16 +-
.../lucene/codecs/blockterms/BlockTermsReader.java | 272 +++--
.../lucene/codecs/blockterms/BlockTermsWriter.java | 107 +-
.../blockterms/FixedGapTermsIndexReader.java | 150 ++-
.../blockterms/FixedGapTermsIndexWriter.java | 65 +-
.../codecs/blockterms/TermsIndexReaderBase.java | 36 +-
.../codecs/blockterms/TermsIndexWriterBase.java | 27 +-
.../blockterms/VariableGapTermsIndexReader.java | 54 +-
.../blockterms/VariableGapTermsIndexWriter.java | 100 +-
.../lucene/codecs/blockterms/package-info.java | 4 +-
.../blocktreeords/BlockTreeOrdsPostingsFormat.java | 33 +-
.../codecs/blocktreeords/FSTOrdsOutputs.java | 56 +-
.../blocktreeords/OrdsBlockTreeTermsReader.java | 115 +-
.../blocktreeords/OrdsBlockTreeTermsWriter.java | 351 +++---
.../codecs/blocktreeords/OrdsFieldReader.java | 61 +-
.../blocktreeords/OrdsIntersectTermsEnum.java | 165 ++-
.../blocktreeords/OrdsIntersectTermsEnumFrame.java | 49 +-
.../codecs/blocktreeords/OrdsSegmentTermsEnum.java | 540 +++++----
.../blocktreeords/OrdsSegmentTermsEnumFrame.java | 191 ++--
.../lucene/codecs/blocktreeords/package-info.java | 5 +-
.../lucene/codecs/bloom/BloomFilterFactory.java | 30 +-
.../codecs/bloom/BloomFilteringPostingsFormat.java | 277 ++---
.../codecs/bloom/DefaultBloomFilterFactory.java | 18 +-
.../org/apache/lucene/codecs/bloom/FuzzySet.java | 255 +++--
.../apache/lucene/codecs/bloom/HashFunction.java | 11 +-
.../apache/lucene/codecs/bloom/MurmurHash2.java | 39 +-
.../apache/lucene/codecs/bloom/package-info.java | 5 +-
.../lucene/codecs/memory/DirectPostingsFormat.java | 409 ++++---
.../lucene/codecs/memory/FSTPostingsFormat.java | 8 +-
.../lucene/codecs/memory/FSTTermOutputs.java | 94 +-
.../lucene/codecs/memory/FSTTermsReader.java | 180 +--
.../lucene/codecs/memory/FSTTermsWriter.java | 142 +--
.../codecs/memory/LegacyBinaryDocValues.java | 15 +-
.../memory/LegacyBinaryDocValuesWrapper.java | 8 +-
.../codecs/memory/LegacyDocValuesIterables.java | 117 +-
.../codecs/memory/LegacyNumericDocValues.java | 6 +-
.../memory/LegacyNumericDocValuesWrapper.java | 7 +-
.../codecs/memory/LegacySortedDocValues.java | 59 +-
.../memory/LegacySortedDocValuesWrapper.java | 8 +-
.../memory/LegacySortedNumericDocValues.java | 28 +-
.../LegacySortedNumericDocValuesWrapper.java | 8 +-
.../codecs/memory/LegacySortedSetDocValues.java | 76 +-
.../memory/LegacySortedSetDocValuesWrapper.java | 8 +-
.../apache/lucene/codecs/memory/package-info.java | 5 +-
.../codecs/simpletext/SimpleTextBKDReader.java | 185 +--
.../codecs/simpletext/SimpleTextBKDWriter.java | 844 ++++++++++----
.../lucene/codecs/simpletext/SimpleTextCodec.java | 18 +-
.../simpletext/SimpleTextCompoundFormat.java | 93 +-
.../simpletext/SimpleTextDocValuesFormat.java | 91 +-
.../simpletext/SimpleTextDocValuesReader.java | 364 +++---
.../simpletext/SimpleTextDocValuesWriter.java | 255 +++--
.../simpletext/SimpleTextFieldInfosFormat.java | 147 ++-
.../codecs/simpletext/SimpleTextFieldsReader.java | 196 ++--
.../codecs/simpletext/SimpleTextFieldsWriter.java | 42 +-
.../simpletext/SimpleTextLiveDocsFormat.java | 61 +-
.../codecs/simpletext/SimpleTextNormsFormat.java | 56 +-
.../codecs/simpletext/SimpleTextPointsFormat.java | 16 +-
.../codecs/simpletext/SimpleTextPointsReader.java | 104 +-
.../codecs/simpletext/SimpleTextPointsWriter.java | 116 +-
.../simpletext/SimpleTextPostingsFormat.java | 16 +-
.../simpletext/SimpleTextSegmentInfoFormat.java | 148 ++-
.../simpletext/SimpleTextStoredFieldsFormat.java | 14 +-
.../simpletext/SimpleTextStoredFieldsReader.java | 93 +-
.../simpletext/SimpleTextStoredFieldsWriter.java | 87 +-
.../simpletext/SimpleTextTermVectorsFormat.java | 14 +-
.../simpletext/SimpleTextTermVectorsReader.java | 58 +-
.../simpletext/SimpleTextTermVectorsWriter.java | 98 +-
.../lucene/codecs/simpletext/SimpleTextUtil.java | 48 +-
.../codecs/simpletext/SimpleTextVectorFormat.java | 14 +-
.../codecs/simpletext/SimpleTextVectorReader.java | 71 +-
.../codecs/simpletext/SimpleTextVectorWriter.java | 22 +-
.../lucene/codecs/simpletext/package-info.java | 4 +-
.../lucene/codecs/uniformsplit/BlockDecoder.java | 10 +-
.../lucene/codecs/uniformsplit/BlockEncoder.java | 18 +-
.../lucene/codecs/uniformsplit/BlockHeader.java | 112 +-
.../lucene/codecs/uniformsplit/BlockLine.java | 160 +--
.../lucene/codecs/uniformsplit/BlockReader.java | 222 ++--
.../lucene/codecs/uniformsplit/BlockWriter.java | 129 ++-
.../uniformsplit/DeltaBaseTermStateSerializer.java | 82 +-
.../lucene/codecs/uniformsplit/FSTDictionary.java | 57 +-
.../lucene/codecs/uniformsplit/FieldMetadata.java | 137 ++-
.../codecs/uniformsplit/IndexDictionary.java | 94 +-
.../codecs/uniformsplit/IntersectBlockReader.java | 233 ++--
.../lucene/codecs/uniformsplit/RamUsageUtil.java | 31 +-
.../lucene/codecs/uniformsplit/TermBytes.java | 59 +-
.../uniformsplit/UniformSplitPostingsFormat.java | 106 +-
.../codecs/uniformsplit/UniformSplitTerms.java | 53 +-
.../uniformsplit/UniformSplitTermsReader.java | 265 +++--
.../uniformsplit/UniformSplitTermsWriter.java | 301 ++---
.../lucene/codecs/uniformsplit/package-info.java | 22 +-
.../uniformsplit/sharedterms/STBlockLine.java | 129 ++-
.../uniformsplit/sharedterms/STBlockReader.java | 44 +-
.../uniformsplit/sharedterms/STBlockWriter.java | 57 +-
.../sharedterms/STIntersectBlockReader.java | 56 +-
.../sharedterms/STMergingBlockReader.java | 52 +-
.../sharedterms/STMergingTermsEnum.java | 37 +-
.../sharedterms/STUniformSplitPostingsFormat.java | 74 +-
.../sharedterms/STUniformSplitTerms.java | 45 +-
.../sharedterms/STUniformSplitTermsReader.java | 100 +-
.../sharedterms/STUniformSplitTermsWriter.java | 256 +++--
.../uniformsplit/sharedterms/package-info.java | 15 +-
.../blockterms/TestFixedGapPostingsFormat.java | 8 +-
.../TestVarGapDocFreqIntervalPostingsFormat.java | 9 +-
.../TestVarGapFixedIntervalPostingsFormat.java | 10 +-
.../codecs/blocktreeords/TestOrdsBlockTree.java | 70 +-
.../codecs/bloom/TestBloomPostingsFormat.java | 5 +-
.../codecs/lucene84/MockTermStateFactory.java | 10 +-
.../codecs/memory/TestDirectPostingsFormat.java | 5 +-
.../codecs/memory/TestFSTPostingsFormat.java | 5 +-
.../simpletext/TestSimpleTextCompoundFormat.java | 1 -
.../simpletext/TestSimpleTextDocValuesFormat.java | 5 +-
.../simpletext/TestSimpleTextFieldInfoFormat.java | 5 +-
.../simpletext/TestSimpleTextNormsFormat.java | 1 -
.../simpletext/TestSimpleTextPointsFormat.java | 5 +-
.../simpletext/TestSimpleTextPostingsFormat.java | 5 +-
.../TestSimpleTextSegmentInfoFormat.java | 9 +-
.../TestSimpleTextStoredFieldsFormat.java | 1 -
.../TestSimpleTextTermVectorsFormat.java | 2 -
.../codecs/uniformsplit/TestBlockWriter.java | 53 +-
.../codecs/uniformsplit/TestFSTDictionary.java | 34 +-
.../lucene/codecs/uniformsplit/TestTermBytes.java | 204 ++--
.../uniformsplit/TestTermBytesComparator.java | 76 +-
.../TestUniformSplitPostingFormat.java | 15 +-
.../sharedterms/TestSTBlockReader.java | 146 +--
.../TestSTUniformSplitPostingFormat.java | 15 +-
.../lucene/codecs/lucene84/Lucene84SkipReader.java | 8 +-
.../java/org/apache/lucene/demo/IndexFiles.java | 103 +-
.../java/org/apache/lucene/demo/SearchFiles.java | 90 +-
.../demo/facet/AssociationsFacetsExample.java | 23 +-
.../lucene/demo/facet/DistanceFacetsExample.java | 113 +-
.../facet/ExpressionAggregationFacetsExample.java | 27 +-
.../facet/MultiCategoryListsFacetsExample.java | 7 +-
.../lucene/demo/facet/RangeFacetsExample.java | 57 +-
.../lucene/demo/facet/SimpleFacetsExample.java | 54 +-
.../demo/facet/SimpleSortedSetFacetsExample.java | 28 +-
.../org/apache/lucene/demo/facet/package-info.java | 4 +-
.../java/org/apache/lucene/demo/package-info.java | 4 +-
.../src/test/org/apache/lucene/demo/TestDemo.java | 11 +-
.../demo/facet/TestAssociationsFacetsExample.java | 19 +-
.../demo/facet/TestDistanceFacetsExample.java | 6 +-
.../TestExpressionAggregationFacetsExample.java | 5 +-
.../facet/TestMultiCategoryListsFacetsExample.java | 10 +-
.../lucene/demo/facet/TestRangeFacetsExample.java | 5 +-
.../lucene/demo/facet/TestSimpleFacetsExample.java | 35 +-
.../facet/TestSimpleSortedSetFacetsExample.java | 14 +-
.../org/apache/lucene/expressions/Bindings.java | 16 +-
.../expressions/CachingExpressionValueSource.java | 34 +-
.../org/apache/lucene/expressions/Expression.java | 24 +-
.../expressions/ExpressionFunctionValues.java | 5 +-
.../lucene/expressions/ExpressionRescorer.java | 25 +-
.../lucene/expressions/ExpressionValueSource.java | 41 +-
.../apache/lucene/expressions/SimpleBindings.java | 36 +-
.../expressions/js/JavascriptBaseVisitor.java | 154 ++-
.../lucene/expressions/js/JavascriptCompiler.java | 258 +++--
.../js/JavascriptErrorHandlingLexer.java | 21 +-
.../js/JavascriptParserErrorStrategy.java | 58 +-
.../lucene/expressions/js/VariableContext.java | 42 +-
.../apache/lucene/expressions/js/package-info.java | 41 +-
.../apache/lucene/expressions/package-info.java | 23 +-
.../lucene/expressions/TestDemoExpressions.java | 184 +--
.../lucene/expressions/TestExpressionRescorer.java | 27 +-
.../expressions/TestExpressionSortField.java | 37 +-
.../lucene/expressions/TestExpressionSorts.java | 42 +-
.../expressions/TestExpressionValidation.java | 78 +-
.../expressions/TestExpressionValueSource.java | 46 +-
.../lucene/expressions/js/TestCustomFunctions.java | 321 ++++--
.../expressions/js/TestJavascriptCompiler.java | 169 +--
.../expressions/js/TestJavascriptFunction.java | 134 +--
.../expressions/js/TestJavascriptOperations.java | 57 +-
.../lucene/expressions/js/TestVariableContext.java | 9 +-
.../org/apache/lucene/facet/DrillDownQuery.java | 78 +-
.../org/apache/lucene/facet/DrillSideways.java | 309 +++--
.../apache/lucene/facet/DrillSidewaysQuery.java | 80 +-
.../apache/lucene/facet/DrillSidewaysScorer.java | 257 +++--
.../java/org/apache/lucene/facet/FacetField.java | 24 +-
.../java/org/apache/lucene/facet/FacetQuery.java | 17 +-
.../java/org/apache/lucene/facet/FacetResult.java | 20 +-
.../src/java/org/apache/lucene/facet/Facets.java | 36 +-
.../org/apache/lucene/facet/FacetsCollector.java | 118 +-
.../lucene/facet/FacetsCollectorManager.java | 16 +-
.../java/org/apache/lucene/facet/FacetsConfig.java | 247 ++--
.../org/apache/lucene/facet/LabelAndValue.java | 3 +-
.../apache/lucene/facet/LongValueFacetCounts.java | 137 ++-
.../org/apache/lucene/facet/MultiFacetQuery.java | 22 +-
.../java/org/apache/lucene/facet/MultiFacets.java | 19 +-
.../facet/RandomSamplingFacetsCollector.java | 140 ++-
.../apache/lucene/facet/TopOrdAndFloatQueue.java | 6 +-
.../org/apache/lucene/facet/TopOrdAndIntQueue.java | 6 +-
.../java/org/apache/lucene/facet/package-info.java | 62 +-
.../org/apache/lucene/facet/range/DoubleRange.java | 83 +-
.../lucene/facet/range/DoubleRangeFacetCounts.java | 73 +-
.../org/apache/lucene/facet/range/LongRange.java | 69 +-
.../lucene/facet/range/LongRangeCounter.java | 74 +-
.../lucene/facet/range/LongRangeFacetCounts.java | 67 +-
.../java/org/apache/lucene/facet/range/Range.java | 6 +-
.../lucene/facet/range/RangeFacetCounts.java | 25 +-
.../apache/lucene/facet/range/package-info.java | 4 +-
.../ConcurrentSortedSetDocValuesFacetCounts.java | 94 +-
.../DefaultSortedSetDocValuesReaderState.java | 62 +-
.../sortedset/SortedSetDocValuesFacetCounts.java | 108 +-
.../sortedset/SortedSetDocValuesFacetField.java | 12 +-
.../sortedset/SortedSetDocValuesReaderState.java | 49 +-
.../lucene/facet/sortedset/package-info.java | 4 +-
.../facet/taxonomy/AssociationFacetField.java | 36 +-
.../facet/taxonomy/CachedOrdinalsReader.java | 52 +-
.../facet/taxonomy/DocValuesOrdinalsReader.java | 22 +-
.../apache/lucene/facet/taxonomy/FacetLabel.java | 74 +-
.../facet/taxonomy/FastTaxonomyFacetCounts.java | 54 +-
.../facet/taxonomy/FloatAssociationFacetField.java | 25 +-
.../lucene/facet/taxonomy/FloatTaxonomyFacets.java | 18 +-
.../facet/taxonomy/IntAssociationFacetField.java | 35 +-
.../lucene/facet/taxonomy/IntTaxonomyFacets.java | 34 +-
.../apache/lucene/facet/taxonomy/LRUHashMap.java | 92 +-
.../facet/taxonomy/OrdinalMappingLeafReader.java | 74 +-
.../lucene/facet/taxonomy/OrdinalsReader.java | 16 +-
.../facet/taxonomy/ParallelTaxonomyArrays.java | 48 +-
.../lucene/facet/taxonomy/PrintTaxonomyStats.java | 30 +-
.../facet/taxonomy/SearcherTaxonomyManager.java | 76 +-
.../lucene/facet/taxonomy/TaxonomyFacetCounts.java | 39 +-
.../lucene/facet/taxonomy/TaxonomyFacetLabels.java | 91 +-
.../TaxonomyFacetSumFloatAssociations.java | 51 +-
.../taxonomy/TaxonomyFacetSumIntAssociations.java | 51 +-
.../taxonomy/TaxonomyFacetSumValueSource.java | 55 +-
.../lucene/facet/taxonomy/TaxonomyFacets.java | 76 +-
.../lucene/facet/taxonomy/TaxonomyMergeUtils.java | 32 +-
.../lucene/facet/taxonomy/TaxonomyReader.java | 202 ++--
.../lucene/facet/taxonomy/TaxonomyWriter.java | 135 +--
.../lucene/facet/taxonomy/directory/Consts.java | 4 +-
.../directory/DirectoryTaxonomyReader.java | 187 +--
.../directory/DirectoryTaxonomyWriter.java | 475 ++++----
.../taxonomy/directory/TaxonomyIndexArrays.java | 89 +-
.../facet/taxonomy/directory/package-info.java | 4 +-
.../apache/lucene/facet/taxonomy/package-info.java | 50 +-
.../facet/taxonomy/writercache/CharBlockArray.java | 20 +-
.../facet/taxonomy/writercache/LabelToOrdinal.java | 28 +-
.../writercache/LruTaxonomyWriterCache.java | 23 +-
.../taxonomy/writercache/NameHashIntCacheLRU.java | 11 +-
.../taxonomy/writercache/NameIntCacheLRU.java | 57 +-
.../taxonomy/writercache/TaxonomyWriterCache.java | 93 +-
.../writercache/UTF8TaxonomyWriterCache.java | 47 +-
.../facet/taxonomy/writercache/package-info.java | 4 +-
.../facet/AssertingSubDocsAtOnceCollector.java | 11 +-
.../org/apache/lucene/facet/FacetTestCase.java | 151 +--
.../org/apache/lucene/facet/SlowDirectory.java | 111 +-
.../apache/lucene/facet/TestDrillDownQuery.java | 67 +-
.../org/apache/lucene/facet/TestDrillSideways.java | 368 +++---
.../org/apache/lucene/facet/TestFacetQuery.java | 23 +-
.../org/apache/lucene/facet/TestFacetsConfig.java | 35 +-
.../lucene/facet/TestLongValueFacetCounts.java | 255 +++--
.../lucene/facet/TestMultipleIndexFields.java | 100 +-
.../lucene/facet/TestParallelDrillSideways.java | 26 +-
.../facet/TestRandomSamplingFacetsCollector.java | 82 +-
.../lucene/facet/range/TestRangeFacetCounts.java | 436 ++++---
.../sortedset/TestSortedSetDocValuesFacets.java | 86 +-
.../facet/taxonomy/TestCachedOrdinalsReader.java | 34 +-
.../lucene/facet/taxonomy/TestFacetLabel.java | 208 ++--
.../taxonomy/TestOrdinalMappingLeafReader.java | 49 +-
.../taxonomy/TestSearcherTaxonomyManager.java | 98 +-
.../facet/taxonomy/TestTaxonomyCombined.java | 756 +++++++------
.../taxonomy/TestTaxonomyFacetAssociations.java | 139 ++-
.../facet/taxonomy/TestTaxonomyFacetCounts.java | 313 +++--
.../facet/taxonomy/TestTaxonomyFacetCounts2.java | 114 +-
.../facet/taxonomy/TestTaxonomyFacetLabels.java | 78 +-
.../taxonomy/TestTaxonomyFacetSumValueSource.java | 169 +--
.../facet/taxonomy/directory/TestAddTaxonomy.java | 114 +-
.../directory/TestBackwardsCompatibility.java | 151 +--
.../directory/TestConcurrentFacetedIndexing.java | 122 +-
.../directory/TestDirectoryTaxonomyReader.java | 231 ++--
.../directory/TestDirectoryTaxonomyWriter.java | 311 ++---
.../writercache/Test2GBCharBlockArray.java | 9 +-
.../taxonomy/writercache/TestCharBlockArray.java | 34 +-
.../writercache/TestLruTaxonomyWriterCache.java | 1 -
.../writercache/TestUTF8TaxonomyWriterCache.java | 7 +-
.../search/join/BaseGlobalOrdinalScorer.java | 5 +-
.../apache/lucene/search/join/BitSetProducer.java | 10 +-
.../lucene/search/join/BlockJoinSelector.java | 131 ++-
.../apache/lucene/search/join/CheckJoinIndex.java | 33 +-
.../search/join/DocValuesTermsCollector.java | 9 +-
.../lucene/search/join/GenericTermsCollector.java | 48 +-
.../search/join/GlobalOrdinalsCollector.java | 8 +-
.../lucene/search/join/GlobalOrdinalsQuery.java | 96 +-
.../join/GlobalOrdinalsWithScoreCollector.java | 28 +-
.../search/join/GlobalOrdinalsWithScoreQuery.java | 119 +-
.../org/apache/lucene/search/join/JoinUtil.java | 633 ++++++-----
.../search/join/ParentChildrenBlockJoinQuery.java | 97 +-
.../search/join/PointInSetIncludingScoreQuery.java | 111 +-
.../lucene/search/join/QueryBitSetProducer.java | 24 +-
.../org/apache/lucene/search/join/ScoreMode.java | 25 +-
.../search/join/SeekingTermSetTermsEnum.java | 13 +-
.../apache/lucene/search/join/TermsCollector.java | 12 +-
.../search/join/TermsIncludingScoreQuery.java | 94 +-
.../org/apache/lucene/search/join/TermsQuery.java | 59 +-
.../search/join/TermsWithScoreCollector.java | 45 +-
.../lucene/search/join/ToChildBlockJoinQuery.java | 81 +-
.../lucene/search/join/ToParentBlockJoinQuery.java | 124 +-
.../search/join/ToParentBlockJoinSortField.java | 96 +-
.../lucene/search/join/ToParentDocValues.java | 132 ++-
.../apache/lucene/search/join/package-info.java | 104 +-
.../apache/lucene/search/join/TestBlockJoin.java | 622 ++++++----
.../lucene/search/join/TestBlockJoinScorer.java | 25 +-
.../lucene/search/join/TestBlockJoinSelector.java | 80 +-
.../lucene/search/join/TestBlockJoinSorting.java | 98 +-
.../search/join/TestBlockJoinValidation.java | 79 +-
.../lucene/search/join/TestCheckJoinIndex.java | 8 +-
.../apache/lucene/search/join/TestJoinUtil.java | 1194 +++++++++++++-------
.../join/TestParentChildrenBlockJoinQuery.java | 21 +-
.../search/join/TestQueryBitSetProducer.java | 34 +-
.../apache/lucene/index/memory/MemoryIndex.java | 730 ++++++------
.../apache/lucene/index/memory/package-info.java | 4 +-
.../lucene/index/memory/TestMemoryIndex.java | 207 ++--
.../memory/TestMemoryIndexAgainstDirectory.java | 287 +++--
798 files changed, 34970 insertions(+), 28748 deletions(-)
[lucene-solr] 01/02: LUCENE-9570: code reformatting [partial].
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 7f8b7ffbcad2265b047a5e2195f76cc924028063
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Sun Jan 3 21:46:01 2021 +0100
LUCENE-9570: code reformatting [partial].
---
dev-tools/scripts/checkJavadocLinks.py | 4 +-
gradle/validation/spotless.gradle | 37 +-
.../analysis/icu/ICUCollationAttributeFactory.java | 82 +-
.../analysis/icu/ICUCollationDocValuesField.java | 31 +-
.../analysis/icu/ICUCollationKeyAnalyzer.java | 66 +-
.../lucene/analysis/icu/ICUFoldingFilter.java | 108 +-
.../analysis/icu/ICUFoldingFilterFactory.java | 10 +-
.../analysis/icu/ICUNormalizer2CharFilter.java | 33 +-
.../icu/ICUNormalizer2CharFilterFactory.java | 38 +-
.../lucene/analysis/icu/ICUNormalizer2Filter.java | 47 +-
.../analysis/icu/ICUNormalizer2FilterFactory.java | 39 +-
.../lucene/analysis/icu/ICUTransformFilter.java | 79 +-
.../analysis/icu/ICUTransformFilterFactory.java | 19 +-
.../apache/lucene/analysis/icu/package-info.java | 4 +-
.../icu/segmentation/BreakIteratorWrapper.java | 19 +-
.../icu/segmentation/CharArrayIterator.java | 12 +-
.../icu/segmentation/CompositeBreakIterator.java | 71 +-
.../segmentation/DefaultICUTokenizerConfig.java | 103 +-
.../analysis/icu/segmentation/ICUTokenizer.java | 105 +-
.../icu/segmentation/ICUTokenizerConfig.java | 12 +-
.../icu/segmentation/ICUTokenizerFactory.java | 91 +-
.../analysis/icu/segmentation/ScriptIterator.java | 86 +-
.../analysis/icu/segmentation/package-info.java | 4 +-
.../ICUCollatedTermAttributeImpl.java | 11 +-
.../icu/tokenattributes/ScriptAttribute.java | 15 +-
.../icu/tokenattributes/ScriptAttributeImpl.java | 26 +-
.../analysis/icu/tokenattributes/package-info.java | 4 +-
.../icu/TestICUCollationDocValuesField.java | 54 +-
.../analysis/icu/TestICUCollationKeyAnalyzer.java | 40 +-
.../lucene/analysis/icu/TestICUFoldingFilter.java | 86 +-
.../analysis/icu/TestICUFoldingFilterFactory.java | 28 +-
.../analysis/icu/TestICUNormalizer2CharFilter.java | 636 ++++++-----
.../icu/TestICUNormalizer2CharFilterFactory.java | 29 +-
.../analysis/icu/TestICUNormalizer2Filter.java | 94 +-
.../icu/TestICUNormalizer2FilterFactory.java | 31 +-
.../analysis/icu/TestICUTransformFilter.java | 98 +-
.../icu/TestICUTransformFilterFactory.java | 40 +-
.../icu/segmentation/TestCharArrayIterator.java | 32 +-
.../icu/segmentation/TestICUTokenizer.java | 552 ++++++---
.../icu/segmentation/TestICUTokenizerCJK.java | 104 +-
.../icu/segmentation/TestICUTokenizerFactory.java | 113 +-
.../icu/segmentation/TestMyanmarSyllable.java | 116 +-
.../icu/segmentation/TestWithCJKBigramFilter.java | 423 ++++---
.../analysis/icu/GenerateUTR30DataFiles.java | 81 +-
.../lucene/analysis/icu/RBBIRuleCompiler.java | 34 +-
.../lucene/analysis/ja/GraphvizFormatter.java | 93 +-
.../lucene/analysis/ja/JapaneseAnalyzer.java | 39 +-
.../lucene/analysis/ja/JapaneseBaseFormFilter.java | 15 +-
.../analysis/ja/JapaneseBaseFormFilterFactory.java | 10 +-
.../ja/JapaneseIterationMarkCharFilter.java | 105 +-
.../ja/JapaneseIterationMarkCharFilterFactory.java | 15 +-
.../analysis/ja/JapaneseKatakanaStemFilter.java | 36 +-
.../ja/JapaneseKatakanaStemFilterFactory.java | 18 +-
.../lucene/analysis/ja/JapaneseNumberFilter.java | 149 ++-
.../analysis/ja/JapaneseNumberFilterFactory.java | 15 +-
.../ja/JapanesePartOfSpeechStopFilter.java | 9 +-
.../ja/JapanesePartOfSpeechStopFilterFactory.java | 13 +-
.../analysis/ja/JapaneseReadingFormFilter.java | 14 +-
.../ja/JapaneseReadingFormFilterFactory.java | 12 +-
.../lucene/analysis/ja/JapaneseTokenizer.java | 736 +++++++-----
.../analysis/ja/JapaneseTokenizerFactory.java | 71 +-
.../java/org/apache/lucene/analysis/ja/Token.java | 90 +-
.../lucene/analysis/ja/dict/BinaryDictionary.java | 127 ++-
.../analysis/ja/dict/CharacterDefinition.java | 44 +-
.../lucene/analysis/ja/dict/ConnectionCosts.java | 22 +-
.../apache/lucene/analysis/ja/dict/Dictionary.java | 35 +-
.../analysis/ja/dict/TokenInfoDictionary.java | 27 +-
.../lucene/analysis/ja/dict/TokenInfoFST.java | 31 +-
.../lucene/analysis/ja/dict/UnknownDictionary.java | 32 +-
.../lucene/analysis/ja/dict/UserDictionary.java | 117 +-
.../lucene/analysis/ja/dict/package-info.java | 4 +-
.../apache/lucene/analysis/ja/package-info.java | 4 +-
.../ja/tokenattributes/BaseFormAttribute.java | 7 +-
.../ja/tokenattributes/BaseFormAttributeImpl.java | 11 +-
.../ja/tokenattributes/InflectionAttribute.java | 8 +-
.../tokenattributes/InflectionAttributeImpl.java | 16 +-
.../ja/tokenattributes/PartOfSpeechAttribute.java | 6 +-
.../tokenattributes/PartOfSpeechAttributeImpl.java | 17 +-
.../ja/tokenattributes/ReadingAttribute.java | 8 +-
.../ja/tokenattributes/ReadingAttributeImpl.java | 16 +-
.../analysis/ja/tokenattributes/package-info.java | 4 +-
.../analysis/ja/util/BinaryDictionaryWriter.java | 115 +-
.../apache/lucene/analysis/ja/util/CSVUtil.java | 58 +-
.../ja/util/CharacterDefinitionWriter.java | 37 +-
.../analysis/ja/util/ConnectionCostsBuilder.java | 6 +-
.../analysis/ja/util/ConnectionCostsWriter.java | 23 +-
.../lucene/analysis/ja/util/DictionaryBuilder.java | 42 +-
.../lucene/analysis/ja/util/ToStringUtil.java | 64 +-
.../ja/util/TokenInfoDictionaryBuilder.java | 46 +-
.../ja/util/TokenInfoDictionaryWriter.java | 10 +-
.../analysis/ja/util/UnknownDictionaryBuilder.java | 30 +-
.../analysis/ja/util/UnknownDictionaryWriter.java | 18 +-
.../lucene/analysis/ja/util/package-info.java | 4 +-
.../analysis/ja/StringMockResourceLoader.java | 2 -
.../org/apache/lucene/analysis/ja/TestCSVUtil.java | 9 +-
.../lucene/analysis/ja/TestExtendedMode.java | 39 +-
.../apache/lucene/analysis/ja/TestFactories.java | 73 +-
.../lucene/analysis/ja/TestJapaneseAnalyzer.java | 221 ++--
.../analysis/ja/TestJapaneseBaseFormFilter.java | 80 +-
.../ja/TestJapaneseBaseFormFilterFactory.java | 36 +-
.../ja/TestJapaneseIterationMarkCharFilter.java | 198 ++--
...TestJapaneseIterationMarkCharFilterFactory.java | 85 +-
.../ja/TestJapaneseKatakanaStemFilter.java | 97 +-
.../ja/TestJapaneseKatakanaStemFilterFactory.java | 63 +-
.../analysis/ja/TestJapaneseNumberFilter.java | 258 ++---
.../ja/TestJapaneseNumberFilterFactory.java | 31 +-
.../TestJapanesePartOfSpeechStopFilterFactory.java | 54 +-
.../analysis/ja/TestJapaneseReadingFormFilter.java | 137 ++-
.../ja/TestJapaneseReadingFormFilterFactory.java | 43 +-
.../lucene/analysis/ja/TestJapaneseTokenizer.java | 686 ++++++-----
.../analysis/ja/TestJapaneseTokenizerFactory.java | 147 ++-
.../apache/lucene/analysis/ja/TestSearchMode.java | 66 +-
.../analysis/ja/dict/TokenInfoDictionaryTest.java | 41 +-
.../analysis/ja/dict/UserDictionaryTest.java | 37 +-
.../lucene/analysis/ja/util/TestToStringUtil.java | 260 +++--
.../analysis/ja/util/UnknownDictionaryTest.java | 17 +-
.../analysis/morfologik/MorfologikAnalyzer.java | 30 +-
.../analysis/morfologik/MorfologikFilter.java | 66 +-
.../morfologik/MorfologikFilterFactory.java | 45 +-
.../morfologik/MorphosyntacticTagsAttribute.java | 20 +-
.../MorphosyntacticTagsAttributeImpl.java | 25 +-
.../lucene/analysis/morfologik/package-info.java | 15 +-
.../analysis/uk/UkrainianMorfologikAnalyzer.java | 60 +-
.../apache/lucene/analysis/uk/package-info.java | 4 +-
.../morfologik/TestMorfologikAnalyzer.java | 110 +-
.../morfologik/TestMorfologikFilterFactory.java | 40 +-
.../lucene/analysis/uk/TestUkrainianAnalyzer.java | 59 +-
.../apache/lucene/analysis/ko/DecompoundToken.java | 20 +-
.../apache/lucene/analysis/ko/DictionaryToken.java | 37 +-
.../lucene/analysis/ko/GraphvizFormatter.java | 95 +-
.../apache/lucene/analysis/ko/KoreanAnalyzer.java | 26 +-
.../lucene/analysis/ko/KoreanNumberFilter.java | 117 +-
.../analysis/ko/KoreanNumberFilterFactory.java | 13 +-
.../analysis/ko/KoreanPartOfSpeechStopFilter.java | 57 +-
.../ko/KoreanPartOfSpeechStopFilterFactory.java | 19 +-
.../analysis/ko/KoreanReadingFormFilter.java | 6 +-
.../ko/KoreanReadingFormFilterFactory.java | 10 +-
.../apache/lucene/analysis/ko/KoreanTokenizer.java | 394 ++++---
.../lucene/analysis/ko/KoreanTokenizerFactory.java | 39 +-
.../java/org/apache/lucene/analysis/ko/POS.java | 197 +---
.../java/org/apache/lucene/analysis/ko/Token.java | 48 +-
.../lucene/analysis/ko/dict/BinaryDictionary.java | 70 +-
.../analysis/ko/dict/CharacterDefinition.java | 35 +-
.../lucene/analysis/ko/dict/ConnectionCosts.java | 20 +-
.../apache/lucene/analysis/ko/dict/Dictionary.java | 41 +-
.../analysis/ko/dict/TokenInfoDictionary.java | 24 +-
.../lucene/analysis/ko/dict/TokenInfoFST.java | 21 +-
.../lucene/analysis/ko/dict/UnknownDictionary.java | 7 +-
.../lucene/analysis/ko/dict/UserDictionary.java | 32 +-
.../lucene/analysis/ko/dict/package-info.java | 4 +-
.../apache/lucene/analysis/ko/package-info.java | 4 +-
.../ko/tokenattributes/PartOfSpeechAttribute.java | 21 +-
.../tokenattributes/PartOfSpeechAttributeImpl.java | 22 +-
.../ko/tokenattributes/ReadingAttribute.java | 13 +-
.../ko/tokenattributes/ReadingAttributeImpl.java | 5 +-
.../analysis/ko/tokenattributes/package-info.java | 4 +-
.../analysis/ko/util/BinaryDictionaryWriter.java | 49 +-
.../apache/lucene/analysis/ko/util/CSVUtil.java | 46 +-
.../ko/util/CharacterDefinitionWriter.java | 36 +-
.../analysis/ko/util/ConnectionCostsBuilder.java | 5 +-
.../analysis/ko/util/ConnectionCostsWriter.java | 22 +-
.../lucene/analysis/ko/util/DictionaryBuilder.java | 24 +-
.../ko/util/TokenInfoDictionaryBuilder.java | 31 +-
.../ko/util/TokenInfoDictionaryWriter.java | 9 +-
.../analysis/ko/util/UnknownDictionaryBuilder.java | 29 +-
.../analysis/ko/util/UnknownDictionaryWriter.java | 15 +-
.../lucene/analysis/ko/util/package-info.java | 4 +-
.../analysis/ko/StringMockResourceLoader.java | 2 -
.../lucene/analysis/ko/TestKoreanAnalyzer.java | 102 +-
.../lucene/analysis/ko/TestKoreanNumberFilter.java | 272 ++---
.../analysis/ko/TestKoreanNumberFilterFactory.java | 28 +-
.../TestKoreanPartOfSpeechStopFilterFactory.java | 31 +-
.../analysis/ko/TestKoreanReadingFormFilter.java | 50 +-
.../ko/TestKoreanReadingFormFilterFactory.java | 29 +-
.../lucene/analysis/ko/TestKoreanTokenizer.java | 814 +++++++------
.../analysis/ko/TestKoreanTokenizerFactory.java | 85 +-
.../analysis/ko/dict/TokenInfoDictionaryTest.java | 39 +-
.../analysis/ko/dict/UserDictionaryTest.java | 6 +-
.../analysis/ko/util/UnknownDictionaryTest.java | 11 +-
.../analysis/opennlp/OpenNLPChunkerFilter.java | 15 +-
.../opennlp/OpenNLPChunkerFilterFactory.java | 6 +-
.../analysis/opennlp/OpenNLPLemmatizerFilter.java | 38 +-
.../opennlp/OpenNLPLemmatizerFilterFactory.java | 20 +-
.../lucene/analysis/opennlp/OpenNLPPOSFilter.java | 12 +-
.../analysis/opennlp/OpenNLPPOSFilterFactory.java | 6 +-
.../opennlp/OpenNLPSentenceBreakIterator.java | 19 +-
.../lucene/analysis/opennlp/OpenNLPTokenizer.java | 28 +-
.../analysis/opennlp/OpenNLPTokenizerFactory.java | 10 +-
.../lucene/analysis/opennlp/package-info.java | 4 +-
.../analysis/opennlp/tools/NLPChunkerOp.java | 8 +-
.../analysis/opennlp/tools/NLPLemmatizerOp.java | 39 +-
.../analysis/opennlp/tools/NLPNERTaggerOp.java | 24 +-
.../analysis/opennlp/tools/NLPPOSTaggerOp.java | 6 +-
.../opennlp/tools/NLPSentenceDetectorOp.java | 6 +-
.../analysis/opennlp/tools/NLPTokenizerOp.java | 4 +-
.../analysis/opennlp/tools/OpenNLPOpsFactory.java | 53 +-
.../analysis/opennlp/tools/package-info.java | 4 +-
.../opennlp/TestOpenNLPChunkerFilterFactory.java | 99 +-
.../TestOpenNLPLemmatizerFilterFactory.java | 332 ++++--
.../opennlp/TestOpenNLPPOSFilterFactory.java | 129 ++-
.../opennlp/TestOpenNLPSentenceBreakIterator.java | 71 +-
.../opennlp/TestOpenNLPTokenizerFactory.java | 95 +-
.../analysis/phonetic/BeiderMorseFilter.java | 34 +-
.../phonetic/BeiderMorseFilterFactory.java | 23 +-
.../phonetic/DaitchMokotoffSoundexFilter.java | 5 +-
.../DaitchMokotoffSoundexFilterFactory.java | 19 +-
.../analysis/phonetic/DoubleMetaphoneFilter.java | 36 +-
.../phonetic/DoubleMetaphoneFilterFactory.java | 14 +-
.../lucene/analysis/phonetic/PhoneticFilter.java | 32 +-
.../analysis/phonetic/PhoneticFilterFactory.java | 64 +-
.../lucene/analysis/phonetic/package-info.java | 4 +-
.../phonetic/DoubleMetaphoneFilterTest.java | 73 +-
.../analysis/phonetic/TestBeiderMorseFilter.java | 167 +--
.../phonetic/TestBeiderMorseFilterFactory.java | 62 +-
.../phonetic/TestDaitchMokotoffSoundexFilter.java | 73 +-
.../TestDaitchMokotoffSoundexFilterFactory.java | 29 +-
.../phonetic/TestDoubleMetaphoneFilterFactory.java | 29 +-
.../analysis/phonetic/TestPhoneticFilter.java | 185 +--
.../phonetic/TestPhoneticFilterFactory.java | 230 ++--
.../lucene/analysis/cn/smart/AnalyzerProfile.java | 35 +-
.../apache/lucene/analysis/cn/smart/CharType.java | 56 +-
.../analysis/cn/smart/HMMChineseTokenizer.java | 19 +-
.../cn/smart/HMMChineseTokenizerFactory.java | 15 +-
.../analysis/cn/smart/SmartChineseAnalyzer.java | 82 +-
.../apache/lucene/analysis/cn/smart/Utility.java | 103 +-
.../lucene/analysis/cn/smart/WordSegmenter.java | 27 +-
.../apache/lucene/analysis/cn/smart/WordType.java | 50 +-
.../analysis/cn/smart/hhmm/AbstractDictionary.java | 66 +-
.../lucene/analysis/cn/smart/hhmm/BiSegGraph.java | 52 +-
.../analysis/cn/smart/hhmm/BigramDictionary.java | 59 +-
.../analysis/cn/smart/hhmm/HHMMSegmenter.java | 45 +-
.../lucene/analysis/cn/smart/hhmm/PathNode.java | 39 +-
.../lucene/analysis/cn/smart/hhmm/SegGraph.java | 30 +-
.../lucene/analysis/cn/smart/hhmm/SegToken.java | 66 +-
.../analysis/cn/smart/hhmm/SegTokenFilter.java | 15 +-
.../analysis/cn/smart/hhmm/SegTokenPair.java | 42 +-
.../analysis/cn/smart/hhmm/WordDictionary.java | 196 ++--
.../analysis/cn/smart/hhmm/package-info.java | 1 +
.../lucene/analysis/cn/smart/package-info.java | 33 +-
.../cn/smart/TestHMMChineseTokenizerFactory.java | 31 +-
.../cn/smart/TestSmartChineseAnalyzer.java | 121 +-
.../apache/lucene/analysis/pl/PolishAnalyzer.java | 76 +-
.../apache/lucene/analysis/pl/package-info.java | 4 +-
.../lucene/analysis/stempel/StempelFilter.java | 32 +-
.../stempel/StempelPolishStemFilterFactory.java | 9 +-
.../lucene/analysis/stempel/StempelStemmer.java | 45 +-
.../lucene/analysis/stempel/package-info.java | 4 +-
.../stempel/src/java/org/egothor/stemmer/Cell.java | 110 +-
.../src/java/org/egothor/stemmer/Compile.java | 159 ++-
.../stempel/src/java/org/egothor/stemmer/Diff.java | 158 ++-
.../src/java/org/egothor/stemmer/DiffIt.java | 125 +-
.../src/java/org/egothor/stemmer/Gener.java | 127 +--
.../stempel/src/java/org/egothor/stemmer/Lift.java | 135 ++-
.../src/java/org/egothor/stemmer/MultiTrie.java | 172 ++-
.../src/java/org/egothor/stemmer/MultiTrie2.java | 179 ++-
.../src/java/org/egothor/stemmer/Optimizer.java | 156 ++-
.../src/java/org/egothor/stemmer/Optimizer2.java | 118 +-
.../src/java/org/egothor/stemmer/Reduce.java | 133 ++-
.../stempel/src/java/org/egothor/stemmer/Row.java | 200 ++--
.../stempel/src/java/org/egothor/stemmer/Trie.java | 252 ++---
.../src/java/org/egothor/stemmer/package-info.java | 4 +-
.../lucene/analysis/pl/TestPolishAnalyzer.java | 14 +-
.../TestStempelPolishStemFilterFactory.java | 30 +-
.../src/test/org/egothor/stemmer/TestCompile.java | 127 +--
.../src/test/org/egothor/stemmer/TestStemmer.java | 189 ++--
.../lucene/backward_codecs/lucene50/ForUtil.java | 92 +-
.../lucene50/Lucene50PostingsFormat.java | 626 +++++-----
.../lucene50/Lucene50PostingsReader.java | 454 +++++---
.../lucene50/Lucene50ScoreSkipReader.java | 60 +-
.../lucene50/Lucene50SkipReader.java | 69 +-
.../lucene50/Lucene50StoredFieldsFormat.java | 139 +--
.../backward_codecs/lucene50/package-info.java | 2 +-
.../lucene60/Lucene60PointsFormat.java | 67 +-
.../lucene60/Lucene60PointsReader.java | 64 +-
.../backward_codecs/lucene60/package-info.java | 2 +-
.../lucene70/Lucene70SegmentInfoFormat.java | 126 ++-
.../backward_codecs/lucene70/package-info.java | 4 +-
.../backward_codecs/lucene80/Lucene80Codec.java | 56 +-
.../backward_codecs/lucene80/package-info.java | 2 +-
.../backward_codecs/lucene84/Lucene84Codec.java | 85 +-
.../backward_codecs/lucene84/package-info.java | 2 +-
.../backward_codecs/lucene86/Lucene86Codec.java | 81 +-
.../backward_codecs/lucene86/package-info.java | 2 +-
.../backward_codecs/lucene87/Lucene87Codec.java | 78 +-
.../backward_codecs/lucene87/package-info.java | 2 +-
.../lucene/backward_codecs/package-info.java | 2 +-
.../lucene50/Lucene50PostingsWriter.java | 121 +-
.../lucene50/Lucene50RWPostingsFormat.java | 12 +-
.../lucene50/Lucene50RWStoredFieldsFormat.java | 20 +-
.../lucene50/Lucene50SkipWriter.java | 68 +-
.../lucene50/TestBlockPostingsFormat.java | 22 +-
.../lucene50/TestBlockPostingsFormat2.java | 30 +-
.../lucene50/TestBlockPostingsFormat3.java | 253 +++--
.../backward_codecs/lucene50/TestForUtil.java | 25 +-
.../lucene50/TestLucene50StoredFieldsFormat.java | 1 -
...tLucene50StoredFieldsFormatHighCompression.java | 28 +-
...estLucene50StoredFieldsFormatMergeInstance.java | 5 +-
.../lucene60/Lucene60PointsWriter.java | 163 +--
.../lucene60/Lucene60RWPointsFormat.java | 2 -
.../lucene60/TestLucene60PointsFormat.java | 212 ++--
.../lucene70/Lucene70RWSegmentInfoFormat.java | 46 +-
.../lucene70/TestLucene70SegmentInfoFormat.java | 4 +-
.../backward_codecs/lucene84/Lucene84RWCodec.java | 7 +-
.../backward_codecs/lucene86/Lucene86RWCodec.java | 7 +-
.../backward_index/TestBackwardsCompatibility.java | 1063 +++++++++--------
.../backward_index/TestIndexWriterOnOldIndex.java | 18 +-
.../backward_index/TestManyPointsInOldIndex.java | 29 +-
.../org/apache/lucene/benchmark/Constants.java | 16 +-
.../apache/lucene/benchmark/byTask/Benchmark.java | 68 +-
.../lucene/benchmark/byTask/PerfRunData.java | 243 ++--
.../benchmark/byTask/feeds/AbstractQueryMaker.java | 24 +-
.../benchmark/byTask/feeds/ContentItemsSource.java | 147 ++-
.../benchmark/byTask/feeds/ContentSource.java | 24 +-
.../benchmark/byTask/feeds/DemoHTMLParser.java | 212 ++--
.../benchmark/byTask/feeds/DirContentSource.java | 63 +-
.../lucene/benchmark/byTask/feeds/DocData.java | 16 +-
.../lucene/benchmark/byTask/feeds/DocMaker.java | 176 ++-
.../byTask/feeds/EnwikiContentSource.java | 95 +-
.../benchmark/byTask/feeds/EnwikiQueryMaker.java | 157 ++-
.../lucene/benchmark/byTask/feeds/FacetSource.java | 15 +-
.../byTask/feeds/FileBasedQueryMaker.java | 57 +-
.../benchmark/byTask/feeds/GeonamesLineParser.java | 13 +-
.../lucene/benchmark/byTask/feeds/HTMLParser.java | 19 +-
.../benchmark/byTask/feeds/LineDocSource.java | 160 +--
.../byTask/feeds/LongToEnglishContentSource.java | 28 +-
.../byTask/feeds/LongToEnglishQueryMaker.java | 17 +-
.../byTask/feeds/NoMoreDataException.java | 11 +-
.../lucene/benchmark/byTask/feeds/QueryMaker.java | 27 +-
.../benchmark/byTask/feeds/RandomFacetSource.java | 38 +-
.../byTask/feeds/ReutersContentSource.java | 34 +-
.../benchmark/byTask/feeds/ReutersQueryMaker.java | 97 +-
.../benchmark/byTask/feeds/SimpleQueryMaker.java | 36 +-
.../byTask/feeds/SimpleSloppyPhraseQueryMaker.java | 21 +-
.../benchmark/byTask/feeds/SingleDocSource.java | 42 +-
.../byTask/feeds/SortableSingleDocSource.java | 272 ++++-
.../benchmark/byTask/feeds/SpatialDocMaker.java | 155 +--
.../byTask/feeds/SpatialFileQueryMaker.java | 32 +-
.../benchmark/byTask/feeds/TrecContentSource.java | 134 ++-
.../benchmark/byTask/feeds/TrecDocParser.java | 110 +-
.../benchmark/byTask/feeds/TrecFBISParser.java | 25 +-
.../benchmark/byTask/feeds/TrecFR94Parser.java | 34 +-
.../benchmark/byTask/feeds/TrecFTParser.java | 19 +-
.../benchmark/byTask/feeds/TrecGov2Parser.java | 17 +-
.../benchmark/byTask/feeds/TrecLATimesParser.java | 33 +-
.../benchmark/byTask/feeds/TrecParserByPath.java | 16 +-
.../benchmark/byTask/feeds/package-info.java | 4 +-
.../lucene/benchmark/byTask/package-info.java | 972 +++++++---------
.../benchmark/byTask/programmatic/Sample.java | 53 +-
.../byTask/programmatic/package-info.java | 4 +-
.../lucene/benchmark/byTask/stats/Points.java | 46 +-
.../lucene/benchmark/byTask/stats/Report.java | 23 +-
.../lucene/benchmark/byTask/stats/TaskStats.java | 99 +-
.../benchmark/byTask/stats/package-info.java | 4 +-
.../lucene/benchmark/byTask/tasks/AddDocTask.java | 24 +-
.../benchmark/byTask/tasks/AddFacetedDocTask.java | 35 +-
.../benchmark/byTask/tasks/AddIndexesTask.java | 41 +-
.../byTask/tasks/AnalyzerFactoryTask.java | 531 +++++----
.../benchmark/byTask/tasks/ClearStatsTask.java | 6 +-
.../benchmark/byTask/tasks/CloseIndexTask.java | 8 +-
.../benchmark/byTask/tasks/CloseReaderTask.java | 12 +-
.../byTask/tasks/CloseTaxonomyIndexTask.java | 7 +-
.../byTask/tasks/CloseTaxonomyReaderTask.java | 11 +-
.../benchmark/byTask/tasks/CommitIndexTask.java | 17 +-
.../byTask/tasks/CommitTaxonomyIndexTask.java | 8 +-
.../byTask/tasks/ConsumeContentSourceTask.java | 6 +-
.../benchmark/byTask/tasks/CreateIndexTask.java | 117 +-
.../byTask/tasks/CreateTaxonomyIndexTask.java | 13 +-
.../benchmark/byTask/tasks/ForceMergeTask.java | 9 +-
.../byTask/tasks/NearRealtimeReaderTask.java | 30 +-
.../benchmark/byTask/tasks/NewAnalyzerTask.java | 69 +-
.../byTask/tasks/NewCollationAnalyzerTask.java | 64 +-
.../benchmark/byTask/tasks/NewLocaleTask.java | 54 +-
.../benchmark/byTask/tasks/NewRoundTask.java | 6 +-
.../benchmark/byTask/tasks/OpenIndexTask.java | 27 +-
.../benchmark/byTask/tasks/OpenReaderTask.java | 16 +-
.../byTask/tasks/OpenTaxonomyIndexTask.java | 10 +-
.../byTask/tasks/OpenTaxonomyReaderTask.java | 7 +-
.../lucene/benchmark/byTask/tasks/PerfTask.java | 164 ++-
.../benchmark/byTask/tasks/PrintReaderTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/ReadTask.java | 83 +-
.../benchmark/byTask/tasks/ReadTokensTask.java | 33 +-
.../benchmark/byTask/tasks/ReopenReaderTask.java | 7 +-
.../lucene/benchmark/byTask/tasks/RepAllTask.java | 27 +-
.../byTask/tasks/RepSelectByPrefTask.java | 36 +-
.../byTask/tasks/RepSumByNameRoundTask.java | 30 +-
.../benchmark/byTask/tasks/RepSumByNameTask.java | 29 +-
.../byTask/tasks/RepSumByPrefRoundTask.java | 33 +-
.../benchmark/byTask/tasks/RepSumByPrefTask.java | 39 +-
.../lucene/benchmark/byTask/tasks/ReportTask.java | 86 +-
.../benchmark/byTask/tasks/ResetInputsTask.java | 11 +-
.../byTask/tasks/ResetSystemEraseTask.java | 13 +-
.../byTask/tasks/ResetSystemSoftTask.java | 11 +-
.../benchmark/byTask/tasks/RollbackIndexTask.java | 5 +-
.../lucene/benchmark/byTask/tasks/SearchTask.java | 9 +-
.../byTask/tasks/SearchTravRetHighlightTask.java | 147 ++-
.../tasks/SearchTravRetLoadFieldSelectorTask.java | 27 +-
.../benchmark/byTask/tasks/SearchTravRetTask.java | 19 +-
.../benchmark/byTask/tasks/SearchTravTask.java | 19 +-
.../byTask/tasks/SearchWithCollectorTask.java | 11 +-
.../benchmark/byTask/tasks/SearchWithSortTask.java | 26 +-
.../lucene/benchmark/byTask/tasks/SetPropTask.java | 28 +-
.../benchmark/byTask/tasks/TaskSequence.java | 159 ++-
.../benchmark/byTask/tasks/UpdateDocTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/WaitTask.java | 20 +-
.../lucene/benchmark/byTask/tasks/WarmTask.java | 15 +-
.../byTask/tasks/WriteEnwikiLineDocTask.java | 22 +-
.../benchmark/byTask/tasks/WriteLineDocTask.java | 129 +--
.../benchmark/byTask/tasks/package-info.java | 4 +-
.../lucene/benchmark/byTask/utils/Algorithm.java | 245 ++--
.../benchmark/byTask/utils/AnalyzerFactory.java | 33 +-
.../lucene/benchmark/byTask/utils/Config.java | 97 +-
.../lucene/benchmark/byTask/utils/Format.java | 17 +-
.../lucene/benchmark/byTask/utils/StreamUtils.java | 37 +-
.../benchmark/byTask/utils/package-info.java | 4 +-
.../org/apache/lucene/benchmark/package-info.java | 51 +-
.../org/apache/lucene/benchmark/quality/Judge.java | 29 +-
.../lucene/benchmark/quality/QualityBenchmark.java | 127 ++-
.../lucene/benchmark/quality/QualityQuery.java | 43 +-
.../benchmark/quality/QualityQueryParser.java | 6 +-
.../lucene/benchmark/quality/QualityStats.java | 183 ++-
.../lucene/benchmark/quality/package-info.java | 66 +-
.../lucene/benchmark/quality/trec/QueryDriver.java | 45 +-
.../benchmark/quality/trec/Trec1MQReader.java | 36 +-
.../lucene/benchmark/quality/trec/TrecJudge.java | 71 +-
.../benchmark/quality/trec/TrecTopicsReader.java | 64 +-
.../benchmark/quality/trec/package-info.java | 4 +-
.../benchmark/quality/utils/DocNameExtractor.java | 52 +-
.../quality/utils/QualityQueriesFinder.java | 92 +-
.../benchmark/quality/utils/SimpleQQParser.java | 19 +-
.../benchmark/quality/utils/SubmissionReport.java | 59 +-
.../benchmark/quality/utils/package-info.java | 4 +-
.../lucene/benchmark/utils/ExtractReuters.java | 32 +-
.../lucene/benchmark/utils/ExtractWikipedia.java | 32 +-
.../lucene/benchmark/utils/package-info.java | 4 +-
.../apache/lucene/benchmark/BenchmarkTestCase.java | 36 +-
.../benchmark/byTask/TestPerfTasksLogic.java | 949 ++++++++--------
.../benchmark/byTask/TestPerfTasksParse.java | 92 +-
.../benchmark/byTask/feeds/DocMakerTest.java | 61 +-
.../byTask/feeds/EnwikiContentSourceTest.java | 144 ++-
.../benchmark/byTask/feeds/LineDocSourceTest.java | 101 +-
.../benchmark/byTask/feeds/TestHtmlParser.java | 93 +-
.../byTask/feeds/TrecContentSourceTest.java | 509 +++++----
.../benchmark/byTask/tasks/AddIndexesTaskTest.java | 47 +-
.../byTask/tasks/CommitIndexTaskTest.java | 5 +-
.../byTask/tasks/CountingSearchTestTask.java | 6 +-
.../byTask/tasks/CreateIndexTaskTest.java | 12 +-
.../benchmark/byTask/tasks/PerfTaskTest.java | 28 +-
.../byTask/tasks/SearchWithSortTaskTest.java | 4 +-
.../byTask/tasks/WriteEnwikiLineDocTaskTest.java | 37 +-
.../byTask/tasks/WriteLineDocTaskTest.java | 106 +-
.../byTask/tasks/alt/AltPackageTaskTest.java | 18 +-
.../benchmark/byTask/tasks/alt/AltTestTask.java | 8 +-
.../benchmark/byTask/utils/StreamUtilsTest.java | 32 +-
.../lucene/benchmark/byTask/utils/TestConfig.java | 3 -
.../lucene/benchmark/quality/TestQualityRun.java | 222 ++--
.../lucene/classification/BM25NBClassifier.java | 64 +-
.../BooleanPerceptronClassifier.java | 104 +-
.../CachingNaiveBayesClassifier.java | 73 +-
.../classification/ClassificationResult.java | 5 +-
.../apache/lucene/classification/Classifier.java | 16 +-
.../classification/KNearestFuzzyClassifier.java | 90 +-
.../classification/KNearestNeighborClassifier.java | 139 +--
.../classification/SimpleNaiveBayesClassifier.java | 90 +-
.../document/DocumentClassifier.java | 36 +-
.../KNearestNeighborDocumentClassifier.java | 72 +-
.../SimpleNaiveBayesDocumentClassifier.java | 84 +-
.../classification/document/package-info.java | 3 +-
.../apache/lucene/classification/package-info.java | 7 +-
.../utils/ConfusionMatrixGenerator.java | 89 +-
.../classification/utils/DatasetSplitter.java | 47 +-
.../utils/DocToDoubleVectorUtils.java | 25 +-
.../classification/utils/NearestFuzzyQuery.java | 101 +-
.../lucene/classification/utils/package-info.java | 4 +-
.../classification/BM25NBClassifierTest.java | 32 +-
.../BooleanPerceptronClassifierTest.java | 26 +-
.../CachingNaiveBayesClassifierTest.java | 48 +-
.../classification/ClassificationTestBase.java | 142 ++-
.../KNearestFuzzyClassifierTest.java | 22 +-
.../KNearestNeighborClassifierTest.java | 103 +-
.../SimpleNaiveBayesClassifierTest.java | 33 +-
.../Test20NewsgroupsClassification.java | 256 ++++-
.../document/DocumentClassificationTestBase.java | 150 +--
.../KNearestNeighborDocumentClassifierTest.java | 164 ++-
.../SimpleNaiveBayesDocumentClassifierTest.java | 116 +-
.../utils/ConfusionMatrixGeneratorTest.java | 97 +-
.../classification/utils/DataSplitterTest.java | 28 +-
.../utils/DocToDoubleVectorUtilsTest.java | 16 +-
.../lucene/codecs/blockterms/BlockTermsReader.java | 272 +++--
.../lucene/codecs/blockterms/BlockTermsWriter.java | 107 +-
.../blockterms/FixedGapTermsIndexReader.java | 150 ++-
.../blockterms/FixedGapTermsIndexWriter.java | 65 +-
.../codecs/blockterms/TermsIndexReaderBase.java | 36 +-
.../codecs/blockterms/TermsIndexWriterBase.java | 27 +-
.../blockterms/VariableGapTermsIndexReader.java | 54 +-
.../blockterms/VariableGapTermsIndexWriter.java | 100 +-
.../lucene/codecs/blockterms/package-info.java | 4 +-
.../blocktreeords/BlockTreeOrdsPostingsFormat.java | 33 +-
.../codecs/blocktreeords/FSTOrdsOutputs.java | 56 +-
.../blocktreeords/OrdsBlockTreeTermsReader.java | 115 +-
.../blocktreeords/OrdsBlockTreeTermsWriter.java | 351 +++---
.../codecs/blocktreeords/OrdsFieldReader.java | 61 +-
.../blocktreeords/OrdsIntersectTermsEnum.java | 165 ++-
.../blocktreeords/OrdsIntersectTermsEnumFrame.java | 49 +-
.../codecs/blocktreeords/OrdsSegmentTermsEnum.java | 540 +++++----
.../blocktreeords/OrdsSegmentTermsEnumFrame.java | 191 ++--
.../lucene/codecs/blocktreeords/package-info.java | 5 +-
.../lucene/codecs/bloom/BloomFilterFactory.java | 30 +-
.../codecs/bloom/BloomFilteringPostingsFormat.java | 277 ++---
.../codecs/bloom/DefaultBloomFilterFactory.java | 18 +-
.../org/apache/lucene/codecs/bloom/FuzzySet.java | 255 +++--
.../apache/lucene/codecs/bloom/HashFunction.java | 11 +-
.../apache/lucene/codecs/bloom/MurmurHash2.java | 39 +-
.../apache/lucene/codecs/bloom/package-info.java | 5 +-
.../lucene/codecs/memory/DirectPostingsFormat.java | 409 ++++---
.../lucene/codecs/memory/FSTPostingsFormat.java | 8 +-
.../lucene/codecs/memory/FSTTermOutputs.java | 94 +-
.../lucene/codecs/memory/FSTTermsReader.java | 180 +--
.../lucene/codecs/memory/FSTTermsWriter.java | 142 +--
.../codecs/memory/LegacyBinaryDocValues.java | 15 +-
.../memory/LegacyBinaryDocValuesWrapper.java | 8 +-
.../codecs/memory/LegacyDocValuesIterables.java | 117 +-
.../codecs/memory/LegacyNumericDocValues.java | 6 +-
.../memory/LegacyNumericDocValuesWrapper.java | 7 +-
.../codecs/memory/LegacySortedDocValues.java | 59 +-
.../memory/LegacySortedDocValuesWrapper.java | 8 +-
.../memory/LegacySortedNumericDocValues.java | 28 +-
.../LegacySortedNumericDocValuesWrapper.java | 8 +-
.../codecs/memory/LegacySortedSetDocValues.java | 76 +-
.../memory/LegacySortedSetDocValuesWrapper.java | 8 +-
.../apache/lucene/codecs/memory/package-info.java | 5 +-
.../codecs/simpletext/SimpleTextBKDReader.java | 185 +--
.../codecs/simpletext/SimpleTextBKDWriter.java | 844 ++++++++++----
.../lucene/codecs/simpletext/SimpleTextCodec.java | 18 +-
.../simpletext/SimpleTextCompoundFormat.java | 93 +-
.../simpletext/SimpleTextDocValuesFormat.java | 91 +-
.../simpletext/SimpleTextDocValuesReader.java | 364 +++---
.../simpletext/SimpleTextDocValuesWriter.java | 255 +++--
.../simpletext/SimpleTextFieldInfosFormat.java | 147 ++-
.../codecs/simpletext/SimpleTextFieldsReader.java | 196 ++--
.../codecs/simpletext/SimpleTextFieldsWriter.java | 42 +-
.../simpletext/SimpleTextLiveDocsFormat.java | 61 +-
.../codecs/simpletext/SimpleTextNormsFormat.java | 56 +-
.../codecs/simpletext/SimpleTextPointsFormat.java | 16 +-
.../codecs/simpletext/SimpleTextPointsReader.java | 104 +-
.../codecs/simpletext/SimpleTextPointsWriter.java | 116 +-
.../simpletext/SimpleTextPostingsFormat.java | 16 +-
.../simpletext/SimpleTextSegmentInfoFormat.java | 148 ++-
.../simpletext/SimpleTextStoredFieldsFormat.java | 14 +-
.../simpletext/SimpleTextStoredFieldsReader.java | 93 +-
.../simpletext/SimpleTextStoredFieldsWriter.java | 87 +-
.../simpletext/SimpleTextTermVectorsFormat.java | 14 +-
.../simpletext/SimpleTextTermVectorsReader.java | 58 +-
.../simpletext/SimpleTextTermVectorsWriter.java | 98 +-
.../lucene/codecs/simpletext/SimpleTextUtil.java | 48 +-
.../codecs/simpletext/SimpleTextVectorFormat.java | 14 +-
.../codecs/simpletext/SimpleTextVectorReader.java | 71 +-
.../codecs/simpletext/SimpleTextVectorWriter.java | 22 +-
.../lucene/codecs/simpletext/package-info.java | 4 +-
.../lucene/codecs/uniformsplit/BlockDecoder.java | 10 +-
.../lucene/codecs/uniformsplit/BlockEncoder.java | 18 +-
.../lucene/codecs/uniformsplit/BlockHeader.java | 112 +-
.../lucene/codecs/uniformsplit/BlockLine.java | 160 +--
.../lucene/codecs/uniformsplit/BlockReader.java | 222 ++--
.../lucene/codecs/uniformsplit/BlockWriter.java | 129 ++-
.../uniformsplit/DeltaBaseTermStateSerializer.java | 82 +-
.../lucene/codecs/uniformsplit/FSTDictionary.java | 57 +-
.../lucene/codecs/uniformsplit/FieldMetadata.java | 137 ++-
.../codecs/uniformsplit/IndexDictionary.java | 94 +-
.../codecs/uniformsplit/IntersectBlockReader.java | 233 ++--
.../lucene/codecs/uniformsplit/RamUsageUtil.java | 31 +-
.../lucene/codecs/uniformsplit/TermBytes.java | 59 +-
.../uniformsplit/UniformSplitPostingsFormat.java | 106 +-
.../codecs/uniformsplit/UniformSplitTerms.java | 53 +-
.../uniformsplit/UniformSplitTermsReader.java | 265 +++--
.../uniformsplit/UniformSplitTermsWriter.java | 301 ++---
.../lucene/codecs/uniformsplit/package-info.java | 22 +-
.../uniformsplit/sharedterms/STBlockLine.java | 129 ++-
.../uniformsplit/sharedterms/STBlockReader.java | 44 +-
.../uniformsplit/sharedterms/STBlockWriter.java | 57 +-
.../sharedterms/STIntersectBlockReader.java | 56 +-
.../sharedterms/STMergingBlockReader.java | 52 +-
.../sharedterms/STMergingTermsEnum.java | 37 +-
.../sharedterms/STUniformSplitPostingsFormat.java | 74 +-
.../sharedterms/STUniformSplitTerms.java | 45 +-
.../sharedterms/STUniformSplitTermsReader.java | 100 +-
.../sharedterms/STUniformSplitTermsWriter.java | 256 +++--
.../uniformsplit/sharedterms/package-info.java | 15 +-
.../blockterms/TestFixedGapPostingsFormat.java | 8 +-
.../TestVarGapDocFreqIntervalPostingsFormat.java | 9 +-
.../TestVarGapFixedIntervalPostingsFormat.java | 10 +-
.../codecs/blocktreeords/TestOrdsBlockTree.java | 70 +-
.../codecs/bloom/TestBloomPostingsFormat.java | 5 +-
.../codecs/lucene84/MockTermStateFactory.java | 10 +-
.../codecs/memory/TestDirectPostingsFormat.java | 5 +-
.../codecs/memory/TestFSTPostingsFormat.java | 5 +-
.../simpletext/TestSimpleTextCompoundFormat.java | 1 -
.../simpletext/TestSimpleTextDocValuesFormat.java | 5 +-
.../simpletext/TestSimpleTextFieldInfoFormat.java | 5 +-
.../simpletext/TestSimpleTextNormsFormat.java | 1 -
.../simpletext/TestSimpleTextPointsFormat.java | 5 +-
.../simpletext/TestSimpleTextPostingsFormat.java | 5 +-
.../TestSimpleTextSegmentInfoFormat.java | 9 +-
.../TestSimpleTextStoredFieldsFormat.java | 1 -
.../TestSimpleTextTermVectorsFormat.java | 2 -
.../codecs/uniformsplit/TestBlockWriter.java | 53 +-
.../codecs/uniformsplit/TestFSTDictionary.java | 34 +-
.../lucene/codecs/uniformsplit/TestTermBytes.java | 204 ++--
.../uniformsplit/TestTermBytesComparator.java | 76 +-
.../TestUniformSplitPostingFormat.java | 15 +-
.../sharedterms/TestSTBlockReader.java | 146 +--
.../TestSTUniformSplitPostingFormat.java | 15 +-
.../lucene/codecs/lucene84/Lucene84SkipReader.java | 8 +-
.../java/org/apache/lucene/demo/IndexFiles.java | 103 +-
.../java/org/apache/lucene/demo/SearchFiles.java | 90 +-
.../demo/facet/AssociationsFacetsExample.java | 23 +-
.../lucene/demo/facet/DistanceFacetsExample.java | 113 +-
.../facet/ExpressionAggregationFacetsExample.java | 27 +-
.../facet/MultiCategoryListsFacetsExample.java | 7 +-
.../lucene/demo/facet/RangeFacetsExample.java | 57 +-
.../lucene/demo/facet/SimpleFacetsExample.java | 54 +-
.../demo/facet/SimpleSortedSetFacetsExample.java | 28 +-
.../org/apache/lucene/demo/facet/package-info.java | 4 +-
.../java/org/apache/lucene/demo/package-info.java | 4 +-
.../src/test/org/apache/lucene/demo/TestDemo.java | 11 +-
.../demo/facet/TestAssociationsFacetsExample.java | 19 +-
.../demo/facet/TestDistanceFacetsExample.java | 6 +-
.../TestExpressionAggregationFacetsExample.java | 5 +-
.../facet/TestMultiCategoryListsFacetsExample.java | 10 +-
.../lucene/demo/facet/TestRangeFacetsExample.java | 5 +-
.../lucene/demo/facet/TestSimpleFacetsExample.java | 35 +-
.../facet/TestSimpleSortedSetFacetsExample.java | 14 +-
.../org/apache/lucene/expressions/Bindings.java | 16 +-
.../expressions/CachingExpressionValueSource.java | 34 +-
.../org/apache/lucene/expressions/Expression.java | 24 +-
.../expressions/ExpressionFunctionValues.java | 5 +-
.../lucene/expressions/ExpressionRescorer.java | 25 +-
.../lucene/expressions/ExpressionValueSource.java | 41 +-
.../apache/lucene/expressions/SimpleBindings.java | 36 +-
.../expressions/js/JavascriptBaseVisitor.java | 154 ++-
.../lucene/expressions/js/JavascriptCompiler.java | 258 +++--
.../js/JavascriptErrorHandlingLexer.java | 21 +-
.../js/JavascriptParserErrorStrategy.java | 58 +-
.../lucene/expressions/js/VariableContext.java | 42 +-
.../apache/lucene/expressions/js/package-info.java | 41 +-
.../apache/lucene/expressions/package-info.java | 23 +-
.../lucene/expressions/TestDemoExpressions.java | 184 +--
.../lucene/expressions/TestExpressionRescorer.java | 27 +-
.../expressions/TestExpressionSortField.java | 37 +-
.../lucene/expressions/TestExpressionSorts.java | 42 +-
.../expressions/TestExpressionValidation.java | 78 +-
.../expressions/TestExpressionValueSource.java | 46 +-
.../lucene/expressions/js/TestCustomFunctions.java | 321 ++++--
.../expressions/js/TestJavascriptCompiler.java | 169 +--
.../expressions/js/TestJavascriptFunction.java | 134 +--
.../expressions/js/TestJavascriptOperations.java | 57 +-
.../lucene/expressions/js/TestVariableContext.java | 9 +-
.../org/apache/lucene/facet/DrillDownQuery.java | 78 +-
.../org/apache/lucene/facet/DrillSideways.java | 309 +++--
.../apache/lucene/facet/DrillSidewaysQuery.java | 80 +-
.../apache/lucene/facet/DrillSidewaysScorer.java | 257 +++--
.../java/org/apache/lucene/facet/FacetField.java | 24 +-
.../java/org/apache/lucene/facet/FacetQuery.java | 17 +-
.../java/org/apache/lucene/facet/FacetResult.java | 20 +-
.../src/java/org/apache/lucene/facet/Facets.java | 36 +-
.../org/apache/lucene/facet/FacetsCollector.java | 118 +-
.../lucene/facet/FacetsCollectorManager.java | 16 +-
.../java/org/apache/lucene/facet/FacetsConfig.java | 247 ++--
.../org/apache/lucene/facet/LabelAndValue.java | 3 +-
.../apache/lucene/facet/LongValueFacetCounts.java | 137 ++-
.../org/apache/lucene/facet/MultiFacetQuery.java | 22 +-
.../java/org/apache/lucene/facet/MultiFacets.java | 19 +-
.../facet/RandomSamplingFacetsCollector.java | 140 ++-
.../apache/lucene/facet/TopOrdAndFloatQueue.java | 6 +-
.../org/apache/lucene/facet/TopOrdAndIntQueue.java | 6 +-
.../java/org/apache/lucene/facet/package-info.java | 62 +-
.../org/apache/lucene/facet/range/DoubleRange.java | 83 +-
.../lucene/facet/range/DoubleRangeFacetCounts.java | 73 +-
.../org/apache/lucene/facet/range/LongRange.java | 69 +-
.../lucene/facet/range/LongRangeCounter.java | 74 +-
.../lucene/facet/range/LongRangeFacetCounts.java | 67 +-
.../java/org/apache/lucene/facet/range/Range.java | 6 +-
.../lucene/facet/range/RangeFacetCounts.java | 25 +-
.../apache/lucene/facet/range/package-info.java | 4 +-
.../ConcurrentSortedSetDocValuesFacetCounts.java | 94 +-
.../DefaultSortedSetDocValuesReaderState.java | 62 +-
.../sortedset/SortedSetDocValuesFacetCounts.java | 108 +-
.../sortedset/SortedSetDocValuesFacetField.java | 12 +-
.../sortedset/SortedSetDocValuesReaderState.java | 49 +-
.../lucene/facet/sortedset/package-info.java | 4 +-
.../facet/taxonomy/AssociationFacetField.java | 36 +-
.../facet/taxonomy/CachedOrdinalsReader.java | 52 +-
.../facet/taxonomy/DocValuesOrdinalsReader.java | 22 +-
.../apache/lucene/facet/taxonomy/FacetLabel.java | 74 +-
.../facet/taxonomy/FastTaxonomyFacetCounts.java | 54 +-
.../facet/taxonomy/FloatAssociationFacetField.java | 25 +-
.../lucene/facet/taxonomy/FloatTaxonomyFacets.java | 18 +-
.../facet/taxonomy/IntAssociationFacetField.java | 35 +-
.../lucene/facet/taxonomy/IntTaxonomyFacets.java | 34 +-
.../apache/lucene/facet/taxonomy/LRUHashMap.java | 92 +-
.../facet/taxonomy/OrdinalMappingLeafReader.java | 74 +-
.../lucene/facet/taxonomy/OrdinalsReader.java | 16 +-
.../facet/taxonomy/ParallelTaxonomyArrays.java | 48 +-
.../lucene/facet/taxonomy/PrintTaxonomyStats.java | 30 +-
.../facet/taxonomy/SearcherTaxonomyManager.java | 76 +-
.../lucene/facet/taxonomy/TaxonomyFacetCounts.java | 39 +-
.../lucene/facet/taxonomy/TaxonomyFacetLabels.java | 91 +-
.../TaxonomyFacetSumFloatAssociations.java | 51 +-
.../taxonomy/TaxonomyFacetSumIntAssociations.java | 51 +-
.../taxonomy/TaxonomyFacetSumValueSource.java | 55 +-
.../lucene/facet/taxonomy/TaxonomyFacets.java | 76 +-
.../lucene/facet/taxonomy/TaxonomyMergeUtils.java | 32 +-
.../lucene/facet/taxonomy/TaxonomyReader.java | 202 ++--
.../lucene/facet/taxonomy/TaxonomyWriter.java | 135 +--
.../lucene/facet/taxonomy/directory/Consts.java | 4 +-
.../directory/DirectoryTaxonomyReader.java | 187 +--
.../directory/DirectoryTaxonomyWriter.java | 475 ++++----
.../taxonomy/directory/TaxonomyIndexArrays.java | 89 +-
.../facet/taxonomy/directory/package-info.java | 4 +-
.../apache/lucene/facet/taxonomy/package-info.java | 50 +-
.../facet/taxonomy/writercache/CharBlockArray.java | 20 +-
.../facet/taxonomy/writercache/LabelToOrdinal.java | 28 +-
.../writercache/LruTaxonomyWriterCache.java | 23 +-
.../taxonomy/writercache/NameHashIntCacheLRU.java | 11 +-
.../taxonomy/writercache/NameIntCacheLRU.java | 57 +-
.../taxonomy/writercache/TaxonomyWriterCache.java | 93 +-
.../writercache/UTF8TaxonomyWriterCache.java | 47 +-
.../facet/taxonomy/writercache/package-info.java | 4 +-
.../facet/AssertingSubDocsAtOnceCollector.java | 11 +-
.../org/apache/lucene/facet/FacetTestCase.java | 151 +--
.../org/apache/lucene/facet/SlowDirectory.java | 111 +-
.../apache/lucene/facet/TestDrillDownQuery.java | 67 +-
.../org/apache/lucene/facet/TestDrillSideways.java | 368 +++---
.../org/apache/lucene/facet/TestFacetQuery.java | 23 +-
.../org/apache/lucene/facet/TestFacetsConfig.java | 35 +-
.../lucene/facet/TestLongValueFacetCounts.java | 255 +++--
.../lucene/facet/TestMultipleIndexFields.java | 100 +-
.../lucene/facet/TestParallelDrillSideways.java | 26 +-
.../facet/TestRandomSamplingFacetsCollector.java | 82 +-
.../lucene/facet/range/TestRangeFacetCounts.java | 436 ++++---
.../sortedset/TestSortedSetDocValuesFacets.java | 86 +-
.../facet/taxonomy/TestCachedOrdinalsReader.java | 34 +-
.../lucene/facet/taxonomy/TestFacetLabel.java | 208 ++--
.../taxonomy/TestOrdinalMappingLeafReader.java | 49 +-
.../taxonomy/TestSearcherTaxonomyManager.java | 98 +-
.../facet/taxonomy/TestTaxonomyCombined.java | 756 +++++++------
.../taxonomy/TestTaxonomyFacetAssociations.java | 139 ++-
.../facet/taxonomy/TestTaxonomyFacetCounts.java | 313 +++--
.../facet/taxonomy/TestTaxonomyFacetCounts2.java | 114 +-
.../facet/taxonomy/TestTaxonomyFacetLabels.java | 78 +-
.../taxonomy/TestTaxonomyFacetSumValueSource.java | 169 +--
.../facet/taxonomy/directory/TestAddTaxonomy.java | 114 +-
.../directory/TestBackwardsCompatibility.java | 151 +--
.../directory/TestConcurrentFacetedIndexing.java | 122 +-
.../directory/TestDirectoryTaxonomyReader.java | 231 ++--
.../directory/TestDirectoryTaxonomyWriter.java | 311 ++---
.../writercache/Test2GBCharBlockArray.java | 9 +-
.../taxonomy/writercache/TestCharBlockArray.java | 34 +-
.../writercache/TestLruTaxonomyWriterCache.java | 1 -
.../writercache/TestUTF8TaxonomyWriterCache.java | 7 +-
.../search/join/BaseGlobalOrdinalScorer.java | 5 +-
.../apache/lucene/search/join/BitSetProducer.java | 10 +-
.../lucene/search/join/BlockJoinSelector.java | 131 ++-
.../apache/lucene/search/join/CheckJoinIndex.java | 33 +-
.../search/join/DocValuesTermsCollector.java | 9 +-
.../lucene/search/join/GenericTermsCollector.java | 48 +-
.../search/join/GlobalOrdinalsCollector.java | 8 +-
.../lucene/search/join/GlobalOrdinalsQuery.java | 96 +-
.../join/GlobalOrdinalsWithScoreCollector.java | 28 +-
.../search/join/GlobalOrdinalsWithScoreQuery.java | 119 +-
.../org/apache/lucene/search/join/JoinUtil.java | 633 ++++++-----
.../search/join/ParentChildrenBlockJoinQuery.java | 97 +-
.../search/join/PointInSetIncludingScoreQuery.java | 111 +-
.../lucene/search/join/QueryBitSetProducer.java | 24 +-
.../org/apache/lucene/search/join/ScoreMode.java | 25 +-
.../search/join/SeekingTermSetTermsEnum.java | 13 +-
.../apache/lucene/search/join/TermsCollector.java | 12 +-
.../search/join/TermsIncludingScoreQuery.java | 94 +-
.../org/apache/lucene/search/join/TermsQuery.java | 59 +-
.../search/join/TermsWithScoreCollector.java | 45 +-
.../lucene/search/join/ToChildBlockJoinQuery.java | 81 +-
.../lucene/search/join/ToParentBlockJoinQuery.java | 124 +-
.../search/join/ToParentBlockJoinSortField.java | 96 +-
.../lucene/search/join/ToParentDocValues.java | 132 ++-
.../apache/lucene/search/join/package-info.java | 104 +-
.../apache/lucene/search/join/TestBlockJoin.java | 622 ++++++----
.../lucene/search/join/TestBlockJoinScorer.java | 25 +-
.../lucene/search/join/TestBlockJoinSelector.java | 80 +-
.../lucene/search/join/TestBlockJoinSorting.java | 98 +-
.../search/join/TestBlockJoinValidation.java | 79 +-
.../lucene/search/join/TestCheckJoinIndex.java | 8 +-
.../apache/lucene/search/join/TestJoinUtil.java | 1194 +++++++++++++-------
.../join/TestParentChildrenBlockJoinQuery.java | 21 +-
.../search/join/TestQueryBitSetProducer.java | 34 +-
.../apache/lucene/index/memory/MemoryIndex.java | 730 ++++++------
.../apache/lucene/index/memory/package-info.java | 4 +-
.../lucene/index/memory/TestMemoryIndex.java | 207 ++--
.../memory/TestMemoryIndexAgainstDirectory.java | 287 +++--
797 files changed, 34968 insertions(+), 28747 deletions(-)
diff --git a/dev-tools/scripts/checkJavadocLinks.py b/dev-tools/scripts/checkJavadocLinks.py
index 6eaff12..1e4e8b5 100644
--- a/dev-tools/scripts/checkJavadocLinks.py
+++ b/dev-tools/scripts/checkJavadocLinks.py
@@ -41,7 +41,7 @@ class FindHyperlinks(HTMLParser):
def handle_starttag(self, tag, attrs):
# NOTE: I don't think 'a' should be in here. But try debugging
# NumericRangeQuery.html. (Could be javadocs bug, it's a generic type...)
- if tag not in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col', 'a', 'dd'):
+ if tag not in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col', 'a', 'dt', 'dd'):
self.stack.append(tag)
if tag == 'a':
id = None
@@ -79,7 +79,7 @@ class FindHyperlinks(HTMLParser):
raise RuntimeError('couldn\'t find an href nor name in link in %s: only got these attrs: %s' % (self.baseURL, attrs))
def handle_endtag(self, tag):
- if tag in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col', 'a', 'dd'):
+ if tag in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col', 'a', 'dt', 'dd'):
return
if len(self.stack) == 0:
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle
index 492f306..3dc3deb 100644
--- a/gradle/validation/spotless.gradle
+++ b/gradle/validation/spotless.gradle
@@ -63,28 +63,41 @@ allprojects { prj ->
"**/tartarus/**"
break
- // Partially complete.
-
- // All others - disable reformatting/ checks for now.
+ case ":lucene:demo":
+ case ":lucene:analysis:morfologik":
case ":lucene:analysis:icu":
case ":lucene:analysis:kuromoji":
- case ":lucene:analysis:morfologik":
+ case ":lucene:memory":
+ case ":lucene:benchmark":
case ":lucene:analysis:nori":
case ":lucene:analysis:opennlp":
case ":lucene:analysis:phonetic":
case ":lucene:analysis:smartcn":
case ":lucene:analysis:stempel":
- case ":lucene:backward-codecs":
- case ":lucene:benchmark":
case ":lucene:classification":
+ case ":lucene:backward-codecs":
case ":lucene:codecs":
- case ":lucene:demo":
+ case ":lucene:join":
+ target "src/**/*.java"
+ targetExclude "**/resources/**"
+ break
+
case ":lucene:expressions":
+ target "src/**/*.java"
+ targetExclude "**/resources/**", "**/JavascriptLexer.java", "**/JavascriptParser.java",
+ "**/JavascriptVisitor.java"
+ break
+
+ // Partially complete.
+
case ":lucene:facet":
+ target "src/**/*.java"
+ targetExclude "**/taxonomy.8.6.3-cfs.zip"
+ break
+
+ // All others - disable reformatting/ checks for now.
case ":lucene:grouping":
- case ":lucene:join":
case ":lucene:luke":
- case ":lucene:memory":
case ":lucene:misc":
case ":lucene:monitor":
case ":lucene:queryparser":
@@ -101,6 +114,12 @@ allprojects { prj ->
}
}
}
+
+ spotlessJava {
+ doFirst {
+ project.mkdir("${buildDir}/spotless/spotlessJava")
+ }
+ }
}
}
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationAttributeFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationAttributeFactory.java
index 3b29cb1..aa9ccc1 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationAttributeFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationAttributeFactory.java
@@ -16,67 +16,55 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Collator;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.collation.CollationAttributeFactory;
import org.apache.lucene.analysis.icu.tokenattributes.ICUCollatedTermAttributeImpl;
+import org.apache.lucene.collation.CollationAttributeFactory;
import org.apache.lucene.util.AttributeFactory;
-import com.ibm.icu.text.Collator;
-
/**
- * <p>
- * Converts each token into its {@link com.ibm.icu.text.CollationKey}, and
- * then encodes bytes as an index term.
- * </p>
- * <p>
- * <strong>WARNING:</strong> Make sure you use exactly the same Collator at
- * index and query time -- CollationKeys are only comparable when produced by
- * the same Collator. {@link com.ibm.icu.text.RuleBasedCollator}s are
- * independently versioned, so it is safe to search against stored
- * CollationKeys if the following are exactly the same (best practice is
- * to store this information with the index and check that they remain the
- * same at query time):
- * </p>
+ * Converts each token into its {@link com.ibm.icu.text.CollationKey}, and then encodes bytes as an
+ * index term.
+ *
+ * <p><strong>WARNING:</strong> Make sure you use exactly the same Collator at index and query time
+ * -- CollationKeys are only comparable when produced by the same Collator. {@link
+ * com.ibm.icu.text.RuleBasedCollator}s are independently versioned, so it is safe to search against
+ * stored CollationKeys if the following are exactly the same (best practice is to store this
+ * information with the index and check that they remain the same at query time):
+ *
* <ol>
- * <li>
- * Collator version - see {@link Collator#getVersion()}
- * </li>
- * <li>
- * The collation strength used - see {@link Collator#setStrength(int)}
- * </li>
- * </ol>
- * <p>
- * CollationKeys generated by ICU Collators are not compatible with those
- * generated by java.text.Collators. Specifically, if you use
- * ICUCollationAttributeFactory to generate index terms, do not use
- * {@link CollationAttributeFactory} on the query side, or vice versa.
- * </p>
- * <p>
- * ICUCollationAttributeFactory is significantly faster and generates significantly
- * shorter keys than CollationAttributeFactory. See
- * <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
- * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
- * generation timing and key length comparisons between ICU4J and
- * java.text.Collator over several languages.
- * </p>
+ * <li>Collator version - see {@link Collator#getVersion()}
+ * <li>The collation strength used - see {@link Collator#setStrength(int)}
+ * </ol>
+ *
+ * <p>CollationKeys generated by ICU Collators are not compatible with those generated by
+ * java.text.Collators. Specifically, if you use ICUCollationAttributeFactory to generate index
+ * terms, do not use {@link CollationAttributeFactory} on the query side, or vice versa.
+ *
+ * <p>ICUCollationAttributeFactory is significantly faster and generates significantly shorter keys
+ * than CollationAttributeFactory. See <a
+ * href="http://site.icu-project.org/charts/collation-icu4j-sun"
+ * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key generation timing and key
+ * length comparisons between ICU4J and java.text.Collator over several languages.
*/
-public class ICUCollationAttributeFactory extends AttributeFactory.StaticImplementationAttributeFactory<ICUCollatedTermAttributeImpl> {
+public class ICUCollationAttributeFactory
+ extends AttributeFactory.StaticImplementationAttributeFactory<ICUCollatedTermAttributeImpl> {
private final Collator collator;
-
+
/**
- * Create an ICUCollationAttributeFactory, using
- * {@link TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY} as the
- * factory for all other attributes.
+ * Create an ICUCollationAttributeFactory, using {@link
+ * TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY} as the factory for all other attributes.
+ *
* @param collator CollationKey generator
*/
public ICUCollationAttributeFactory(Collator collator) {
this(TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, collator);
}
-
+
/**
- * Create an ICUCollationAttributeFactory, using the supplied Attribute
- * Factory as the factory for all other attributes.
+ * Create an ICUCollationAttributeFactory, using the supplied Attribute Factory as the factory for
+ * all other attributes.
+ *
* @param delegate Attribute Factory
* @param collator CollationKey generator
*/
@@ -84,7 +72,7 @@ public class ICUCollationAttributeFactory extends AttributeFactory.StaticImpleme
super(delegate, ICUCollatedTermAttributeImpl.class);
this.collator = collator;
}
-
+
@Override
public ICUCollatedTermAttributeImpl createInstance() {
return new ICUCollatedTermAttributeImpl(collator);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationDocValuesField.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationDocValuesField.java
index a7901cc..a1fb1b9 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationDocValuesField.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationDocValuesField.java
@@ -16,40 +16,37 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.text.RawCollationKey;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.util.BytesRef;
-import com.ibm.icu.text.Collator;
-import com.ibm.icu.text.RawCollationKey;
-
/**
* Indexes collation keys as a single-valued {@link SortedDocValuesField}.
- * <p>
- * This is more efficient that {@link ICUCollationKeyAnalyzer} if the field
- * only has one value: no uninversion is necessary to sort on the field,
- * locale-sensitive range queries can still work via {@code DocValuesRangeQuery},
- * and the underlying data structures built at index-time are likely more efficient
- * and use less memory than FieldCache.
+ *
+ * <p>This is more efficient that {@link ICUCollationKeyAnalyzer} if the field only has one value:
+ * no uninversion is necessary to sort on the field, locale-sensitive range queries can still work
+ * via {@code DocValuesRangeQuery}, and the underlying data structures built at index-time are
+ * likely more efficient and use less memory than FieldCache.
*/
public final class ICUCollationDocValuesField extends Field {
private final String name;
private final Collator collator;
private final BytesRef bytes = new BytesRef();
private final RawCollationKey key = new RawCollationKey();
-
+
/**
* Create a new ICUCollationDocValuesField.
- * <p>
- * NOTE: you should not create a new one for each document, instead
- * just make one and reuse it during your indexing process, setting
- * the value via {@link #setStringValue(String)}.
+ *
+ * <p>NOTE: you should not create a new one for each document, instead just make one and reuse it
+ * during your indexing process, setting the value via {@link #setStringValue(String)}.
+ *
* @param name field name
* @param collator Collator for generating collation keys.
*/
// TODO: can we make this trap-free? maybe just synchronize on the collator
- // instead?
+ // instead?
public ICUCollationDocValuesField(String name, Collator collator) {
super(name, SortedDocValuesField.TYPE);
this.name = name;
@@ -65,7 +62,7 @@ public final class ICUCollationDocValuesField extends Field {
public String name() {
return name;
}
-
+
@Override
public void setStringValue(String value) {
collator.getRawCollationKey(value, key);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationKeyAnalyzer.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationKeyAnalyzer.java
index dff5699..a7abe87 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationKeyAnalyzer.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUCollationKeyAnalyzer.java
@@ -16,52 +16,36 @@
*/
package org.apache.lucene.analysis.icu;
-
-
import com.ibm.icu.text.Collator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.collation.CollationKeyAnalyzer;
/**
- * <p>
- * Configures {@link KeywordTokenizer} with {@link ICUCollationAttributeFactory}.
- * <p>
- * Converts the token into its {@link com.ibm.icu.text.CollationKey}, and
- * then encodes the CollationKey directly to allow it to
- * be stored as an index term.
- * </p>
- * <p>
- * <strong>WARNING:</strong> Make sure you use exactly the same Collator at
- * index and query time -- CollationKeys are only comparable when produced by
- * the same Collator. {@link com.ibm.icu.text.RuleBasedCollator}s are
- * independently versioned, so it is safe to search against stored
- * CollationKeys if the following are exactly the same (best practice is
- * to store this information with the index and check that they remain the
- * same at query time):
- * </p>
+ * Configures {@link KeywordTokenizer} with {@link ICUCollationAttributeFactory}.
+ *
+ * <p>Converts the token into its {@link com.ibm.icu.text.CollationKey}, and then encodes the
+ * CollationKey directly to allow it to be stored as an index term.
+ *
+ * <p><strong>WARNING:</strong> Make sure you use exactly the same Collator at index and query time
+ * -- CollationKeys are only comparable when produced by the same Collator. {@link
+ * com.ibm.icu.text.RuleBasedCollator}s are independently versioned, so it is safe to search against
+ * stored CollationKeys if the following are exactly the same (best practice is to store this
+ * information with the index and check that they remain the same at query time):
+ *
* <ol>
- * <li>
- * Collator version - see {@link Collator#getVersion()}
- * </li>
- * <li>
- * The collation strength used - see {@link Collator#setStrength(int)}
- * </li>
- * </ol>
- * <p>
- * CollationKeys generated by ICU Collators are not compatible with those
- * generated by java.text.Collators. Specifically, if you use
- * ICUCollationKeyAnalyzer to generate index terms, do not use
- * {@link CollationKeyAnalyzer} on the query side, or vice versa.
- * </p>
- * <p>
- * ICUCollationKeyAnalyzer is significantly faster and generates significantly
- * shorter keys than CollationKeyAnalyzer. See
- * <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
- * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
- * generation timing and key length comparisons between ICU4J and
- * java.text.Collator over several languages.
- * </p>
+ * <li>Collator version - see {@link Collator#getVersion()}
+ * <li>The collation strength used - see {@link Collator#setStrength(int)}
+ * </ol>
+ *
+ * <p>CollationKeys generated by ICU Collators are not compatible with those generated by
+ * java.text.Collators. Specifically, if you use ICUCollationKeyAnalyzer to generate index terms, do
+ * not use {@link CollationKeyAnalyzer} on the query side, or vice versa.
+ *
+ * <p>ICUCollationKeyAnalyzer is significantly faster and generates significantly shorter keys than
+ * CollationKeyAnalyzer. See <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
+ * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key generation timing and key
+ * length comparisons between ICU4J and java.text.Collator over several languages.
*
* @since 3.1
*/
@@ -77,10 +61,10 @@ public final class ICUCollationKeyAnalyzer extends Analyzer {
this.factory = new ICUCollationAttributeFactory(collator);
}
-
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- KeywordTokenizer tokenizer = new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+ KeywordTokenizer tokenizer =
+ new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
return new TokenStreamComponents(tokenizer, tokenizer);
}
}
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilter.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilter.java
index 9c3770c..9daa181 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilter.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilter.java
@@ -16,76 +16,72 @@
*/
package org.apache.lucene.analysis.icu;
-
-import org.apache.lucene.analysis.TokenStream;
-
import com.ibm.icu.text.Normalizer2;
+import org.apache.lucene.analysis.TokenStream;
/**
- * A TokenFilter that applies search term folding to Unicode text,
- * applying foldings from UTR#30 Character Foldings.
- * <p>
- * This filter applies the following foldings from the report to unicode text:
+ * A TokenFilter that applies search term folding to Unicode text, applying foldings from UTR#30
+ * Character Foldings.
+ *
+ * <p>This filter applies the following foldings from the report to unicode text:
+ *
* <ul>
- * <li>Accent removal
- * <li>Case folding
- * <li>Canonical duplicates folding
- * <li>Dashes folding
- * <li>Diacritic removal (including stroke, hook, descender)
- * <li>Greek letterforms folding
- * <li>Han Radical folding
- * <li>Hebrew Alternates folding
- * <li>Jamo folding
- * <li>Letterforms folding
- * <li>Math symbol folding
- * <li>Multigraph Expansions: All
- * <li>Native digit folding
- * <li>No-break folding
- * <li>Overline folding
- * <li>Positional forms folding
- * <li>Small forms folding
- * <li>Space folding
- * <li>Spacing Accents folding
- * <li>Subscript folding
- * <li>Superscript folding
- * <li>Suzhou Numeral folding
- * <li>Symbol folding
- * <li>Underline folding
- * <li>Vertical forms folding
- * <li>Width folding
+ * <li>Accent removal
+ * <li>Case folding
+ * <li>Canonical duplicates folding
+ * <li>Dashes folding
+ * <li>Diacritic removal (including stroke, hook, descender)
+ * <li>Greek letterforms folding
+ * <li>Han Radical folding
+ * <li>Hebrew Alternates folding
+ * <li>Jamo folding
+ * <li>Letterforms folding
+ * <li>Math symbol folding
+ * <li>Multigraph Expansions: All
+ * <li>Native digit folding
+ * <li>No-break folding
+ * <li>Overline folding
+ * <li>Positional forms folding
+ * <li>Small forms folding
+ * <li>Space folding
+ * <li>Spacing Accents folding
+ * <li>Subscript folding
+ * <li>Superscript folding
+ * <li>Suzhou Numeral folding
+ * <li>Symbol folding
+ * <li>Underline folding
+ * <li>Vertical forms folding
+ * <li>Width folding
* </ul>
- * <p>
- * Additionally, Default Ignorables are removed, and text is normalized to NFKC.
- * All foldings, case folding, and normalization mappings are applied recursively
- * to ensure a fully folded and normalized result.
- * </p>
- * <p>
- * A normalizer with additional settings such as a filter that lists characters not
- * to be normalized can be passed in the constructor.
- * </p>
+ *
+ * <p>Additionally, Default Ignorables are removed, and text is normalized to NFKC. All foldings,
+ * case folding, and normalization mappings are applied recursively to ensure a fully folded and
+ * normalized result.
+ *
+ * <p>A normalizer with additional settings such as a filter that lists characters not to be
+ * normalized can be passed in the constructor.
*/
public final class ICUFoldingFilter extends ICUNormalizer2Filter {
/**
- * A normalizer for search term folding to Unicode text,
- * applying foldings from UTR#30 Character Foldings.
+ * A normalizer for search term folding to Unicode text, applying foldings from UTR#30 Character
+ * Foldings.
*/
- public static final Normalizer2 NORMALIZER = Normalizer2.getInstance(
- // TODO: if the wrong version of the ICU jar is used, loading these data files may give a strange error.
- // maybe add an explicit check? http://icu-project.org/apiref/icu4j/com/ibm/icu/util/VersionInfo.html
- ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"),
- "utr30", Normalizer2.Mode.COMPOSE);
+ public static final Normalizer2 NORMALIZER =
+ Normalizer2.getInstance(
+ // TODO: if the wrong version of the ICU jar is used, loading these data files may give a
+ // strange error.
+ // maybe add an explicit check?
+ // http://icu-project.org/apiref/icu4j/com/ibm/icu/util/VersionInfo.html
+ ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"),
+ "utr30",
+ Normalizer2.Mode.COMPOSE);
- /**
- * Create a new ICUFoldingFilter on the specified input
- */
+ /** Create a new ICUFoldingFilter on the specified input */
public ICUFoldingFilter(TokenStream input) {
super(input, NORMALIZER);
}
- /**
- * Create a new ICUFoldingFilter on the specified input with the specified
- * normalizer
- */
+ /** Create a new ICUFoldingFilter on the specified input with the specified normalizer */
public ICUFoldingFilter(TokenStream input, Normalizer2 normalizer) {
super(input, normalizer);
}
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
index cf2e516..f8e84e9 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
@@ -16,17 +16,16 @@
*/
package org.apache.lucene.analysis.icu;
-
-import java.util.Map;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
-import org.apache.lucene.analysis.TokenStream;
+import java.util.Map;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link ICUFoldingFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_folded" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -34,6 +33,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.ICUFoldingFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 3.1.0
* @lucene.spi {@value #NAME}
*/
@@ -45,7 +45,7 @@ public class ICUFoldingFilterFactory extends TokenFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUFoldingFilterFactory */
- public ICUFoldingFilterFactory(Map<String,String> args) {
+ public ICUFoldingFilterFactory(Map<String, String> args) {
super(args);
Normalizer2 normalizer = ICUFoldingFilter.NORMALIZER;
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java
index 07b1c88..2992a99 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilter.java
@@ -16,19 +16,14 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Normalizer2;
import java.io.IOException;
import java.io.Reader;
import java.util.Objects;
-
import org.apache.lucene.analysis.CharacterUtils;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
-import com.ibm.icu.text.Normalizer2;
-
-/**
- * Normalize token text with ICU's {@link Normalizer2}.
- */
+/** Normalize token text with ICU's {@link Normalizer2}. */
public final class ICUNormalizer2CharFilter extends BaseCharFilter {
private final Normalizer2 normalizer;
@@ -40,10 +35,9 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
private int checkedInputBoundary;
private int charCount;
-
/**
- * Create a new Normalizer2CharFilter that combines NFKC normalization, Case
- * Folding, and removes Default Ignorables (NFKC_Casefold)
+ * Create a new Normalizer2CharFilter that combines NFKC normalization, Case Folding, and removes
+ * Default Ignorables (NFKC_Casefold)
*/
public ICUNormalizer2CharFilter(Reader in) {
this(in, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
@@ -51,13 +45,14 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
/**
* Create a new Normalizer2CharFilter with the specified Normalizer2
+ *
* @param in text
* @param normalizer normalizer to use
*/
public ICUNormalizer2CharFilter(Reader in, Normalizer2 normalizer) {
this(in, normalizer, 128);
}
-
+
// for testing ONLY
ICUNormalizer2CharFilter(Reader in, Normalizer2 normalizer, int bufferSize) {
super(in);
@@ -110,7 +105,8 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
break;
}
- final int lastCodePoint = Character.codePointBefore(tmpBuffer.getBuffer(), tmpBuffer.getLength(), 0);
+ final int lastCodePoint =
+ Character.codePointBefore(tmpBuffer.getBuffer(), tmpBuffer.getLength(), 0);
if (normalizer.isInert(lastCodePoint)) {
// we require an inert char so that we can normalize content before and
// after this character independently
@@ -133,7 +129,7 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
if (resLen > 0) return resLen;
}
int resLen = readFromIoNormalizeUptoBoundary();
- if(resLen > 0){
+ if (resLen > 0) {
afterQuickCheckYes = false;
}
return resLen;
@@ -162,8 +158,8 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
while (checkedInputBoundary <= bufLen - 1) {
int charLen = Character.charCount(inputBuffer.codePointAt(checkedInputBoundary));
checkedInputBoundary += charLen;
- if (checkedInputBoundary < bufLen && normalizer.hasBoundaryBefore(inputBuffer
- .codePointAt(checkedInputBoundary))) {
+ if (checkedInputBoundary < bufLen
+ && normalizer.hasBoundaryBefore(inputBuffer.codePointAt(checkedInputBoundary))) {
foundBoundary = true;
break;
}
@@ -182,8 +178,7 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
private int normalizeInputUpto(final int length) {
final int destOrigLen = resultBuffer.length();
- normalizer.normalizeSecondAndAppend(resultBuffer,
- inputBuffer.subSequence(0, length));
+ normalizer.normalizeSecondAndAppend(resultBuffer, inputBuffer.subSequence(0, length));
inputBuffer.delete(0, length);
checkedInputBoundary = Math.max(checkedInputBoundary - length, 0);
final int resultLength = resultBuffer.length() - destOrigLen;
@@ -199,7 +194,7 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
final int diff = inputLength - outputLength;
final int cumuDiff = getLastCumulativeDiff();
if (diff < 0) {
- for (int i = 1; i <= -diff; ++i) {
+ for (int i = 1; i <= -diff; ++i) {
addOffCorrectMap(charCount + i, cumuDiff - i);
}
} else {
@@ -216,4 +211,4 @@ public final class ICUNormalizer2CharFilter extends BaseCharFilter {
}
return len;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
index 390507c..ed0d1f0 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
@@ -16,32 +16,31 @@
*/
package org.apache.lucene.analysis.icu;
-
-import java.io.Reader;
-import java.util.Arrays;
-import java.util.Map;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.Map;
import org.apache.lucene.analysis.CharFilterFactory;
/**
* Factory for {@link ICUNormalizer2CharFilter}
- * <p>
- * Supports the following attributes:
+ *
+ * <p>Supports the following attributes:
+ *
* <ul>
- * <li>form: A <a href="http://unicode.org/reports/tr15/">Unicode Normalization Form</a>,
- * one of 'nfc','nfkc', 'nfkc_cf'. Default is nfkc_cf.
- * <li>mode: Either 'compose' or 'decompose'. Default is compose. Use "decompose" with nfc
- * or nfkc, to get nfd or nfkd, respectively.
- * <li>filter: A {@link UnicodeSet} pattern. Codepoints outside the set are
- * always left unchanged. Default is [] (the null set, no filtering).
+ * <li>form: A <a href="http://unicode.org/reports/tr15/">Unicode Normalization Form</a>, one of
+ * 'nfc','nfkc', 'nfkc_cf'. Default is nfkc_cf.
+ * <li>mode: Either 'compose' or 'decompose'. Default is compose. Use "decompose" with nfc or
+ * nfkc, to get nfd or nfkd, respectively.
+ * <li>filter: A {@link UnicodeSet} pattern. Codepoints outside the set are always left unchanged.
+ * Default is [] (the null set, no filtering).
* </ul>
+ *
* @see ICUNormalizer2CharFilter
* @see Normalizer2
* @see FilteredNormalizer2
- *
* @since 4.10.0
* @lucene.spi {@value #NAME}
*/
@@ -53,13 +52,16 @@ public class ICUNormalizer2CharFilterFactory extends CharFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUNormalizer2CharFilterFactory */
- public ICUNormalizer2CharFilterFactory(Map<String,String> args) {
+ public ICUNormalizer2CharFilterFactory(Map<String, String> args) {
super(args);
String form = get(args, "form", "nfkc_cf");
String mode = get(args, "mode", Arrays.asList("compose", "decompose"), "compose");
- Normalizer2 normalizer = Normalizer2.getInstance
- (null, form, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
-
+ Normalizer2 normalizer =
+ Normalizer2.getInstance(
+ null,
+ form,
+ "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
+
String filter = get(args, "filter");
if (filter != null) {
UnicodeSet set = new UnicodeSet(filter);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2Filter.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2Filter.java
index 000d7c2..6c50011 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2Filter.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2Filter.java
@@ -16,39 +16,37 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Normalizer;
+import com.ibm.icu.text.Normalizer2;
import java.io.IOException;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import com.ibm.icu.text.Normalizer;
-import com.ibm.icu.text.Normalizer2;
-
/**
* Normalize token text with ICU's {@link com.ibm.icu.text.Normalizer2}
- * <p>
- * With this filter, you can normalize text in the following ways:
+ *
+ * <p>With this filter, you can normalize text in the following ways:
+ *
* <ul>
- * <li> NFKC Normalization, Case Folding, and removing Ignorables (the default)
- * <li> Using a standard Normalization mode (NFC, NFD, NFKC, NFKD)
- * <li> Based on rules from a custom normalization mapping.
+ * <li>NFKC Normalization, Case Folding, and removing Ignorables (the default)
+ * <li>Using a standard Normalization mode (NFC, NFD, NFKC, NFKD)
+ * <li>Based on rules from a custom normalization mapping.
* </ul>
- * <p>
- * If you use the defaults, this filter is a simple way to standardize Unicode text
- * in a language-independent way for search:
+ *
+ * <p>If you use the defaults, this filter is a simple way to standardize Unicode text in a
+ * language-independent way for search:
+ *
* <ul>
- * <li> The case folding that it does can be seen as a replacement for
- * LowerCaseFilter: For example, it handles cases such as the Greek sigma, so that
- * "Μάϊος" and "ΜΆΪΟΣ" will match correctly.
- * <li> The normalization will standardizes different forms of the same
- * character in Unicode. For example, CJK full-width numbers will be standardized
- * to their ASCII forms.
- * <li> Ignorables such as Zero-Width Joiner and Variation Selectors are removed.
- * These are typically modifier characters that affect display.
+ * <li>The case folding that it does can be seen as a replacement for LowerCaseFilter: For
+ * example, it handles cases such as the Greek sigma, so that "Μάϊος" and "ΜΆΪΟΣ" will match
+ * correctly.
+ * <li>The normalization will standardizes different forms of the same character in Unicode. For
+ * example, CJK full-width numbers will be standardized to their ASCII forms.
+ * <li>Ignorables such as Zero-Width Joiner and Variation Selectors are removed. These are
+ * typically modifier characters that affect display.
* </ul>
- *
+ *
* @see com.ibm.icu.text.Normalizer2
* @see com.ibm.icu.text.FilteredNormalizer2
*/
@@ -58,8 +56,8 @@ public class ICUNormalizer2Filter extends TokenFilter {
private final StringBuilder buffer = new StringBuilder();
/**
- * Create a new Normalizer2Filter that combines NFKC normalization, Case
- * Folding, and removes Default Ignorables (NFKC_Casefold)
+ * Create a new Normalizer2Filter that combines NFKC normalization, Case Folding, and removes
+ * Default Ignorables (NFKC_Casefold)
*/
public ICUNormalizer2Filter(TokenStream input) {
this(input, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
@@ -67,6 +65,7 @@ public class ICUNormalizer2Filter extends TokenFilter {
/**
* Create a new Normalizer2Filter with the specified Normalizer2
+ *
* @param input stream
* @param normalizer normalizer to use
*/
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
index 1385bfa..722771e 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
@@ -16,28 +16,28 @@
*/
package org.apache.lucene.analysis.icu;
-
-import java.util.Arrays;
-import java.util.Map;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
-import org.apache.lucene.analysis.TokenStream;
+import java.util.Arrays;
+import java.util.Map;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link ICUNormalizer2Filter}
- * <p>
- * Supports the following attributes:
+ *
+ * <p>Supports the following attributes:
+ *
* <ul>
- * <li>form: A <a href="http://unicode.org/reports/tr15/">Unicode Normalization Form</a>,
- * one of 'nfc','nfkc', 'nfkc_cf'. Default is nfkc_cf.
- * <li>mode: Either 'compose' or 'decompose'. Default is compose. Use "decompose" with nfc
- * or nfkc, to get nfd or nfkd, respectively.
- * <li>filter: A {@link UnicodeSet} pattern. Codepoints outside the set are
- * always left unchanged. Default is [] (the null set, no filtering).
+ * <li>form: A <a href="http://unicode.org/reports/tr15/">Unicode Normalization Form</a>, one of
+ * 'nfc','nfkc', 'nfkc_cf'. Default is nfkc_cf.
+ * <li>mode: Either 'compose' or 'decompose'. Default is compose. Use "decompose" with nfc or
+ * nfkc, to get nfd or nfkd, respectively.
+ * <li>filter: A {@link UnicodeSet} pattern. Codepoints outside the set are always left unchanged.
+ * Default is [] (the null set, no filtering).
* </ul>
+ *
* @see ICUNormalizer2Filter
* @see Normalizer2
* @see FilteredNormalizer2
@@ -52,13 +52,16 @@ public class ICUNormalizer2FilterFactory extends TokenFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUNormalizer2FilterFactory */
- public ICUNormalizer2FilterFactory(Map<String,String> args) {
+ public ICUNormalizer2FilterFactory(Map<String, String> args) {
super(args);
String form = get(args, "form", "nfkc_cf");
String mode = get(args, "mode", Arrays.asList("compose", "decompose"), "compose");
- Normalizer2 normalizer = Normalizer2.getInstance
- (null, form, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
-
+ Normalizer2 normalizer =
+ Normalizer2.getInstance(
+ null,
+ form,
+ "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);
+
String filter = get(args, "filter");
if (filter != null) {
UnicodeSet set = new UnicodeSet(filter);
@@ -79,7 +82,7 @@ public class ICUNormalizer2FilterFactory extends TokenFilterFactory {
}
// TODO: support custom normalization
-
+
@Override
public TokenStream create(TokenStream input) {
return new ICUNormalizer2Filter(input, normalizer);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java
index 35f1410..a615226 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java
@@ -16,42 +16,43 @@
*/
package org.apache.lucene.analysis.icu;
-
-import java.io.IOException;
-
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-
import com.ibm.icu.text.Replaceable;
import com.ibm.icu.text.Transliterator;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;
+import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that transforms text with ICU.
- * <p>
- * ICU provides text-transformation functionality via its Transliteration API.
- * Although script conversion is its most common use, a Transliterator can
- * actually perform a more general class of tasks. In fact, Transliterator
- * defines a very general API which specifies only that a segment of the input
- * text is replaced by new text. The particulars of this conversion are
- * determined entirely by subclasses of Transliterator.
- * </p>
- * <p>
- * Some useful transformations for search are built-in:
+ *
+ * <p>ICU provides text-transformation functionality via its Transliteration API. Although script
+ * conversion is its most common use, a Transliterator can actually perform a more general class of
+ * tasks. In fact, Transliterator defines a very general API which specifies only that a segment of
+ * the input text is replaced by new text. The particulars of this conversion are determined
+ * entirely by subclasses of Transliterator.
+ *
+ * <p>Some useful transformations for search are built-in:
+ *
* <ul>
- * <li>Conversion from Traditional to Simplified Chinese characters
- * <li>Conversion from Hiragana to Katakana
- * <li>Conversion from Fullwidth to Halfwidth forms.
- * <li>Script conversions, for example Serbian Cyrillic to Latin
+ * <li>Conversion from Traditional to Simplified Chinese characters
+ * <li>Conversion from Hiragana to Katakana
+ * <li>Conversion from Fullwidth to Halfwidth forms.
+ * <li>Script conversions, for example Serbian Cyrillic to Latin
* </ul>
- * <p>
- * Example usage: <blockquote>stream = new ICUTransformFilter(stream,
- * Transliterator.getInstance("Traditional-Simplified"));</blockquote>
+ *
+ * <p>Example usage:
+ *
+ * <blockquote>
+ *
+ * stream = new ICUTransformFilter(stream, Transliterator.getInstance("Traditional-Simplified"));
+ *
+ * </blockquote>
+ *
* <br>
- * For more details, see the <a
- * href="http://userguide.icu-project.org/transforms/general">ICU User
+ * For more details, see the <a href="http://userguide.icu-project.org/transforms/general">ICU User
* Guide</a>.
*/
public final class ICUTransformFilter extends TokenFilter {
@@ -69,7 +70,7 @@ public final class ICUTransformFilter extends TokenFilter {
/**
* Create a new ICUTransformFilter that transforms text on the given stream.
- *
+ *
* @param input {@link TokenStream} to filter.
* @param transform Transliterator to transform the text.
*/
@@ -78,14 +79,14 @@ public final class ICUTransformFilter extends TokenFilter {
super(input);
this.transform = transform;
- /*
+ /*
* This is cheating, but speeds things up a lot.
* If we wanted to use pkg-private APIs we could probably do better.
*/
- if (transform.getFilter() == null && transform instanceof com.ibm.icu.text.RuleBasedTransliterator) {
+ if (transform.getFilter() == null
+ && transform instanceof com.ibm.icu.text.RuleBasedTransliterator) {
final UnicodeSet sourceSet = transform.getSourceSet();
- if (sourceSet != null && !sourceSet.isEmpty())
- transform.setFilter(sourceSet);
+ if (sourceSet != null && !sourceSet.isEmpty()) transform.setFilter(sourceSet);
}
}
@@ -96,8 +97,8 @@ public final class ICUTransformFilter extends TokenFilter {
*/
if (input.incrementToken()) {
replaceableAttribute.setText(termAtt);
-
- final int length = termAtt.length();
+
+ final int length = termAtt.length();
position.start = 0;
position.limit = length;
position.contextStart = 0;
@@ -109,10 +110,8 @@ public final class ICUTransformFilter extends TokenFilter {
return false;
}
}
-
- /**
- * Wrap a {@link CharTermAttribute} with the Replaceable API.
- */
+
+ /** Wrap a {@link CharTermAttribute} with the Replaceable API. */
static final class ReplaceableTermAttribute implements Replaceable {
private char buffer[];
private int length;
@@ -166,8 +165,7 @@ public final class ICUTransformFilter extends TokenFilter {
}
@Override
- public void replace(int start, int limit, char[] text, int charsStart,
- int charsLen) {
+ public void replace(int start, int limit, char[] text, int charsStart, int charsLen) {
// shift text if necessary for the replacement
final int newLength = shiftForReplace(start, limit, charsLen);
// insert the replacement text
@@ -180,8 +178,7 @@ public final class ICUTransformFilter extends TokenFilter {
final int replacementLength = limit - start;
final int newLength = length - replacementLength + charsLen;
// resize if necessary
- if (newLength > length)
- buffer = token.resizeBuffer(newLength);
+ if (newLength > length) buffer = token.resizeBuffer(newLength);
// if the substring being replaced is longer or shorter than the
// replacement, need to shift things around
if (replacementLength != charsLen && limit < length)
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
index e757013..3880c4a 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
@@ -16,22 +16,22 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Transliterator;
import java.util.Arrays;
import java.util.Map;
-
-import com.ibm.icu.text.Transliterator;
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link ICUTransformFilter}.
- * <p>
- * Supports the following attributes:
+ *
+ * <p>Supports the following attributes:
+ *
* <ul>
* <li>id (mandatory): A Transliterator ID, one from {@link Transliterator#getAvailableIDs()}
* <li>direction (optional): Either 'forward' or 'reverse'. Default is forward.
* </ul>
+ *
* @see Transliterator
* @since 3.1.0
* @lucene.spi {@value #NAME}
@@ -42,13 +42,14 @@ public class ICUTransformFilterFactory extends TokenFilterFactory {
public static final String NAME = "icuTransform";
private final Transliterator transliterator;
-
+
// TODO: add support for custom rules
/** Creates a new ICUTransformFilterFactory */
- public ICUTransformFilterFactory(Map<String,String> args) {
+ public ICUTransformFilterFactory(Map<String, String> args) {
super(args);
String id = require(args, "id");
- String direction = get(args, "direction", Arrays.asList("forward", "reverse"), "forward", false);
+ String direction =
+ get(args, "direction", Arrays.asList("forward", "reverse"), "forward", false);
int dir = "forward".equals(direction) ? Transliterator.FORWARD : Transliterator.REVERSE;
transliterator = Transliterator.getInstance(id, dir);
if (!args.isEmpty()) {
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/package-info.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/package-info.java
index a396db4..4a3d975 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/package-info.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analysis components based on ICU
- */
+/** Analysis components based on ICU */
package org.apache.lucene.analysis.icu;
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/BreakIteratorWrapper.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/BreakIteratorWrapper.java
index e21d2fd..bbbc8a9 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/BreakIteratorWrapper.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/BreakIteratorWrapper.java
@@ -22,8 +22,9 @@ import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;
/**
- * Wraps RuleBasedBreakIterator, making object reuse convenient and
- * emitting a rule status for emoji sequences.
+ * Wraps RuleBasedBreakIterator, making object reuse convenient and emitting a rule status for emoji
+ * sequences.
+ *
* @lucene.experimental
*/
final class BreakIteratorWrapper {
@@ -32,11 +33,11 @@ final class BreakIteratorWrapper {
private char text[];
private int start;
private int status;
-
+
BreakIteratorWrapper(RuleBasedBreakIterator rbbi) {
this.rbbi = rbbi;
}
-
+
int current() {
return rbbi.current();
}
@@ -51,10 +52,11 @@ final class BreakIteratorWrapper {
status = calcStatus(current, next);
return next;
}
-
+
/** Returns current rule status for the text between breaks. (determines token type) */
private int calcStatus(int current, int next) {
- // to support presentation selectors, we need to handle alphanum, num, and none at least, so currently not worth optimizing.
+ // to support presentation selectors, we need to handle alphanum, num, and none at least, so
+ // currently not worth optimizing.
// https://unicode.org/cldr/utility/list-unicodeset.jsp?a=%5B%3AEmoji%3A%5D-%5B%3AEmoji_Presentation%3A%5D&g=Word_Break&i=
if (next != BreakIterator.DONE && isEmoji(current, next)) {
return ICUTokenizerConfig.EMOJI_SEQUENCE_STATUS;
@@ -62,7 +64,7 @@ final class BreakIteratorWrapper {
return rbbi.getRuleStatus();
}
}
-
+
// See unicode doc L2/16-315 for rationale.
// basically for us the ambiguous cases (keycap/etc) as far as types go.
static final UnicodeSet EMOJI_RK = new UnicodeSet("[\u002a\u00230-9©®™〰〽]").freeze();
@@ -76,7 +78,8 @@ final class BreakIteratorWrapper {
int codepoint = UTF16.charAt(text, 0, end, begin);
if (EMOJI.contains(codepoint)) {
if (EMOJI_RK.contains(codepoint)) {
- // if its in EmojiRK, we don't treat it as emoji unless there is evidence it forms emoji sequence,
+ // if its in EmojiRK, we don't treat it as emoji unless there is evidence it forms emoji
+ // sequence,
// an emoji presentation selector or keycap follows.
int trailer = begin + Character.charCount(codepoint);
return trailer < end && (text[trailer] == 0xFE0F || text[trailer] == 0x20E3);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java
index 3a8ee76..3f8af81 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java
@@ -16,11 +16,11 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import java.text.CharacterIterator;
/**
* Wraps a char[] as CharacterIterator for processing with a BreakIterator
+ *
* @lucene.experimental
*/
final class CharArrayIterator implements CharacterIterator {
@@ -30,21 +30,21 @@ final class CharArrayIterator implements CharacterIterator {
private int length;
private int limit;
- public char [] getText() {
+ public char[] getText() {
return array;
}
-
+
public int getStart() {
return start;
}
-
+
public int getLength() {
return length;
}
-
+
/**
* Set a new region of text to be examined by this iterator
- *
+ *
* @param array text buffer to examine
* @param start offset into buffer
* @param length maximum length to examine
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java
index 3cb39ed..8759ead 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java
@@ -16,31 +16,31 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.lang.UProperty;
import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.BreakIterator;
/**
- * An internal BreakIterator for multilingual text, following recommendations
- * from: UAX #29: Unicode Text Segmentation. (http://unicode.org/reports/tr29/)
- * <p>
- * See http://unicode.org/reports/tr29/#Tailoring for the motivation of this
- * design.
- * <p>
- * Text is first divided into script boundaries. The processing is then
- * delegated to the appropriate break iterator for that specific script.
- * <p>
- * This break iterator also allows you to retrieve the ISO 15924 script code
- * associated with a piece of text.
- * <p>
- * See also UAX #29, UTR #24
+ * An internal BreakIterator for multilingual text, following recommendations from: UAX #29: Unicode
+ * Text Segmentation. (http://unicode.org/reports/tr29/)
+ *
+ * <p>See http://unicode.org/reports/tr29/#Tailoring for the motivation of this design.
+ *
+ * <p>Text is first divided into script boundaries. The processing is then delegated to the
+ * appropriate break iterator for that specific script.
+ *
+ * <p>This break iterator also allows you to retrieve the ISO 15924 script code associated with a
+ * piece of text.
+ *
+ * <p>See also UAX #29, UTR #24
+ *
* @lucene.experimental
*/
final class CompositeBreakIterator {
private final ICUTokenizerConfig config;
- private final BreakIteratorWrapper wordBreakers[] = new BreakIteratorWrapper[1 + UCharacter.getIntPropertyMaxValue(UProperty.SCRIPT)];
+ private final BreakIteratorWrapper wordBreakers[] =
+ new BreakIteratorWrapper[1 + UCharacter.getIntPropertyMaxValue(UProperty.SCRIPT)];
private BreakIteratorWrapper rbbi;
private final ScriptIterator scriptIterator;
@@ -53,38 +53,41 @@ final class CompositeBreakIterator {
}
/**
- * Retrieve the next break position. If the RBBI range is exhausted within the
- * script boundary, examine the next script boundary.
- *
+ * Retrieve the next break position. If the RBBI range is exhausted within the script boundary,
+ * examine the next script boundary.
+ *
* @return the next break position or BreakIterator.DONE
*/
int next() {
int next = rbbi.next();
while (next == BreakIterator.DONE && scriptIterator.next()) {
rbbi = getBreakIterator(scriptIterator.getScriptCode());
- rbbi.setText(text, scriptIterator.getScriptStart(),
+ rbbi.setText(
+ text,
+ scriptIterator.getScriptStart(),
scriptIterator.getScriptLimit() - scriptIterator.getScriptStart());
next = rbbi.next();
}
- return (next == BreakIterator.DONE) ? BreakIterator.DONE : next
- + scriptIterator.getScriptStart();
+ return (next == BreakIterator.DONE)
+ ? BreakIterator.DONE
+ : next + scriptIterator.getScriptStart();
}
/**
* Retrieve the current break position.
- *
+ *
* @return the current break position or BreakIterator.DONE
*/
int current() {
final int current = rbbi.current();
- return (current == BreakIterator.DONE) ? BreakIterator.DONE : current
- + scriptIterator.getScriptStart();
+ return (current == BreakIterator.DONE)
+ ? BreakIterator.DONE
+ : current + scriptIterator.getScriptStart();
}
/**
- * Retrieve the rule status code (token type) from the underlying break
- * iterator
- *
+ * Retrieve the rule status code (token type) from the underlying break iterator
+ *
* @return rule status code (see RuleBasedBreakIterator constants)
*/
int getRuleStatus() {
@@ -92,9 +95,9 @@ final class CompositeBreakIterator {
}
/**
- * Retrieve the UScript script code for the current token. This code can be
- * decoded with UScript into a name or ISO 15924 code.
- *
+ * Retrieve the UScript script code for the current token. This code can be decoded with UScript
+ * into a name or ISO 15924 code.
+ *
* @return UScript script code for the current token.
*/
int getScriptCode() {
@@ -103,7 +106,7 @@ final class CompositeBreakIterator {
/**
* Set a new region of text to be examined by this iterator
- *
+ *
* @param text buffer of text
* @param start offset into buffer
* @param length maximum length to examine
@@ -113,14 +116,16 @@ final class CompositeBreakIterator {
scriptIterator.setText(text, start, length);
if (scriptIterator.next()) {
rbbi = getBreakIterator(scriptIterator.getScriptCode());
- rbbi.setText(text, scriptIterator.getScriptStart(),
+ rbbi.setText(
+ text,
+ scriptIterator.getScriptStart(),
scriptIterator.getScriptLimit() - scriptIterator.getScriptStart());
} else {
rbbi = getBreakIterator(UScript.COMMON);
rbbi.setText(text, 0, 0);
}
}
-
+
private BreakIteratorWrapper getBreakIterator(int scriptCode) {
if (wordBreakers[scriptCode] == null)
wordBreakers[scriptCode] = new BreakIteratorWrapper(config.getBreakIterator(scriptCode));
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java
index 10e6c67..9f815a6 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java
@@ -16,81 +16,86 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.lucene.analysis.standard.StandardTokenizer;
-
import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.BreakIterator;
import com.ibm.icu.text.RuleBasedBreakIterator;
import com.ibm.icu.util.ULocale;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
/**
- * Default {@link ICUTokenizerConfig} that is generally applicable
- * to many languages.
- * <p>
- * Generally tokenizes Unicode text according to UAX#29
- * ({@link BreakIterator#getWordInstance(ULocale) BreakIterator.getWordInstance(ULocale.ROOT)}),
- * but with the following tailorings:
+ * Default {@link ICUTokenizerConfig} that is generally applicable to many languages.
+ *
+ * <p>Generally tokenizes Unicode text according to UAX#29 ({@link
+ * BreakIterator#getWordInstance(ULocale) BreakIterator.getWordInstance(ULocale.ROOT)}), but with
+ * the following tailorings:
+ *
* <ul>
- * <li>Thai, Lao, Myanmar, Khmer, and CJK text is broken into words with a dictionary.
+ * <li>Thai, Lao, Myanmar, Khmer, and CJK text is broken into words with a dictionary.
* </ul>
+ *
* @lucene.experimental
*/
public class DefaultICUTokenizerConfig extends ICUTokenizerConfig {
/** Token type for words containing ideographic characters */
- public static final String WORD_IDEO = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.IDEOGRAPHIC];
+ public static final String WORD_IDEO =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.IDEOGRAPHIC];
/** Token type for words containing Japanese hiragana */
- public static final String WORD_HIRAGANA = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HIRAGANA];
+ public static final String WORD_HIRAGANA =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HIRAGANA];
/** Token type for words containing Japanese katakana */
- public static final String WORD_KATAKANA = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.KATAKANA];
- /** Token type for words containing Korean hangul */
+ public static final String WORD_KATAKANA =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.KATAKANA];
+ /** Token type for words containing Korean hangul */
public static final String WORD_HANGUL = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HANGUL];
/** Token type for words that contain letters */
- public static final String WORD_LETTER = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.ALPHANUM];
+ public static final String WORD_LETTER =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.ALPHANUM];
/** Token type for words that appear to be numbers */
public static final String WORD_NUMBER = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.NUM];
/** Token type for words that appear to be emoji sequences */
public static final String WORD_EMOJI = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.EMOJI];
-
+
/*
* the default breakiterators in use. these can be expensive to
* instantiate, cheap to clone.
- */
+ */
// we keep the cjk breaking separate, thats because it cannot be customized (because dictionary
- // is only triggered when kind = WORD, but kind = LINE by default and we have no non-evil way to change it)
+ // is only triggered when kind = WORD, but kind = LINE by default and we have no non-evil way to
+ // change it)
private static final BreakIterator cjkBreakIterator = BreakIterator.getWordInstance(ULocale.ROOT);
- // TODO: if the wrong version of the ICU jar is used, loading these data files may give a strange error.
- // maybe add an explicit check? http://icu-project.org/apiref/icu4j/com/ibm/icu/util/VersionInfo.html
+ // TODO: if the wrong version of the ICU jar is used, loading these data files may give a strange
+ // error.
+ // maybe add an explicit check?
+ // http://icu-project.org/apiref/icu4j/com/ibm/icu/util/VersionInfo.html
// the same as ROOT, except no dictionary segmentation for cjk
- private static final RuleBasedBreakIterator defaultBreakIterator =
- readBreakIterator("Default.brk");
- private static final RuleBasedBreakIterator myanmarSyllableIterator =
- readBreakIterator("MyanmarSyllable.brk");
-
+ private static final RuleBasedBreakIterator defaultBreakIterator =
+ readBreakIterator("Default.brk");
+ private static final RuleBasedBreakIterator myanmarSyllableIterator =
+ readBreakIterator("MyanmarSyllable.brk");
+
// TODO: deprecate this boolean? you only care if you are doing super-expert stuff...
private final boolean cjkAsWords;
private final boolean myanmarAsWords;
-
- /**
- * Creates a new config. This object is lightweight, but the first
- * time the class is referenced, breakiterators will be initialized.
- * @param cjkAsWords true if cjk text should undergo dictionary-based segmentation,
- * otherwise text will be segmented according to UAX#29 defaults.
- * If this is true, all Han+Hiragana+Katakana words will be tagged as
- * IDEOGRAPHIC.
+
+ /**
+ * Creates a new config. This object is lightweight, but the first time the class is referenced,
+ * breakiterators will be initialized.
+ *
+ * @param cjkAsWords true if cjk text should undergo dictionary-based segmentation, otherwise text
+ * will be segmented according to UAX#29 defaults. If this is true, all Han+Hiragana+Katakana
+ * words will be tagged as IDEOGRAPHIC.
* @param myanmarAsWords true if Myanmar text should undergo dictionary-based segmentation,
- * otherwise it will be tokenized as syllables.
+ * otherwise it will be tokenized as syllables.
*/
- public DefaultICUTokenizerConfig(boolean cjkAsWords, boolean myanmarAsWords) {
+ public DefaultICUTokenizerConfig(boolean cjkAsWords, boolean myanmarAsWords) {
this.cjkAsWords = cjkAsWords;
this.myanmarAsWords = myanmarAsWords;
}
-
+
@Override
public boolean combineCJ() {
return cjkAsWords;
@@ -98,15 +103,17 @@ public class DefaultICUTokenizerConfig extends ICUTokenizerConfig {
@Override
public RuleBasedBreakIterator getBreakIterator(int script) {
- switch(script) {
- case UScript.JAPANESE: return (RuleBasedBreakIterator)cjkBreakIterator.clone();
- case UScript.MYANMAR:
+ switch (script) {
+ case UScript.JAPANESE:
+ return (RuleBasedBreakIterator) cjkBreakIterator.clone();
+ case UScript.MYANMAR:
if (myanmarAsWords) {
- return (RuleBasedBreakIterator)defaultBreakIterator.clone();
+ return (RuleBasedBreakIterator) defaultBreakIterator.clone();
} else {
- return (RuleBasedBreakIterator)myanmarSyllableIterator.clone();
+ return (RuleBasedBreakIterator) myanmarSyllableIterator.clone();
}
- default: return (RuleBasedBreakIterator)defaultBreakIterator.clone();
+ default:
+ return (RuleBasedBreakIterator) defaultBreakIterator.clone();
}
}
@@ -129,11 +136,9 @@ public class DefaultICUTokenizerConfig extends ICUTokenizerConfig {
}
private static RuleBasedBreakIterator readBreakIterator(String filename) {
- InputStream is =
- DefaultICUTokenizerConfig.class.getResourceAsStream(filename);
+ InputStream is = DefaultICUTokenizerConfig.class.getResourceAsStream(filename);
try {
- RuleBasedBreakIterator bi =
- RuleBasedBreakIterator.getInstanceFromCompiledRules(is);
+ RuleBasedBreakIterator bi = RuleBasedBreakIterator.getInstanceFromCompiledRules(is);
is.close();
return bi;
} catch (IOException e) {
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
index 8b62ddb..5e35f6a 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
@@ -16,27 +16,24 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.text.BreakIterator;
import java.io.IOException;
import java.io.Reader;
-
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeFactory;
-import com.ibm.icu.lang.UCharacter;
-import com.ibm.icu.text.BreakIterator;
-
/**
* Breaks text into words according to UAX #29: Unicode Text Segmentation
* (http://www.unicode.org/reports/tr29/)
- * <p>
- * Words are broken across script boundaries, then segmented according to
- * the BreakIterator and typing provided by the {@link ICUTokenizerConfig}
- * </p>
+ *
+ * <p>Words are broken across script boundaries, then segmented according to the BreakIterator and
+ * typing provided by the {@link ICUTokenizerConfig}
+ *
* @see ICUTokenizerConfig
* @lucene.experimental
*/
@@ -44,11 +41,11 @@ public final class ICUTokenizer extends Tokenizer {
private static final int IOBUFFER = 4096;
private final char buffer[] = new char[IOBUFFER];
/** true length of text in the buffer */
- private int length = 0;
+ private int length = 0;
/** length in buffer that can be evaluated safely, up to a safe end point */
- private int usableLength = 0;
+ private int usableLength = 0;
/** accumulated offset of previous buffers for this reader, for offsetAtt */
- private int offset = 0;
+ private int offset = 0;
private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */
private final ICUTokenizerConfig config;
@@ -58,13 +55,12 @@ public final class ICUTokenizer extends Tokenizer {
private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class);
/**
- * Construct a new ICUTokenizer that breaks text into words from the given
- * Reader.
- * <p>
- * The default script-specific handling is used.
- * <p>
- * The default attribute factory is used.
- *
+ * Construct a new ICUTokenizer that breaks text into words from the given Reader.
+ *
+ * <p>The default script-specific handling is used.
+ *
+ * <p>The default attribute factory is used.
+ *
* @see DefaultICUTokenizerConfig
*/
public ICUTokenizer() {
@@ -72,10 +68,10 @@ public final class ICUTokenizer extends Tokenizer {
}
/**
- * Construct a new ICUTokenizer that breaks text into words from the given
- * Reader, using a tailored BreakIterator configuration.
- * <p>
- * The default attribute factory is used.
+ * Construct a new ICUTokenizer that breaks text into words from the given Reader, using a
+ * tailored BreakIterator configuration.
+ *
+ * <p>The default attribute factory is used.
*
* @param config Tailored BreakIterator configuration
*/
@@ -84,8 +80,8 @@ public final class ICUTokenizer extends Tokenizer {
}
/**
- * Construct a new ICUTokenizer that breaks text into words from the given
- * Reader, using a tailored BreakIterator configuration.
+ * Construct a new ICUTokenizer that breaks text into words from the given Reader, using a
+ * tailored BreakIterator configuration.
*
* @param factory AttributeFactory to use
* @param config Tailored BreakIterator configuration
@@ -99,39 +95,38 @@ public final class ICUTokenizer extends Tokenizer {
@Override
public boolean incrementToken() throws IOException {
clearAttributes();
- if (length == 0)
- refill();
+ if (length == 0) refill();
while (!incrementTokenBuffer()) {
refill();
if (length <= 0) // no more bytes to read;
- return false;
+ return false;
}
return true;
}
-
+
@Override
public void reset() throws IOException {
super.reset();
breaker.setText(buffer, 0, 0);
length = usableLength = offset = 0;
}
-
+
@Override
public void end() throws IOException {
super.end();
final int finalOffset = (length < 0) ? offset : offset + length;
offsetAtt.setOffset(correctOffset(finalOffset), correctOffset(finalOffset));
- }
+ }
/*
- * This tokenizes text based upon the longest matching rule, and because of
+ * This tokenizes text based upon the longest matching rule, and because of
* this, isn't friendly to a Reader.
- *
+ *
* Text is read from the input stream in 4kB chunks. Within a 4kB chunk of
* text, the last unambiguous break point is found (in this implementation:
* white space character) Any remaining characters represent possible partial
* words, so are appended to the front of the next chunk.
- *
+ *
* There is the possibility that there are no unambiguous break points within
* an entire 4kB chunk of text (binary data). So there is a maximum word limit
* of 4kB since it will not try to grow the buffer in this case.
@@ -139,20 +134,22 @@ public final class ICUTokenizer extends Tokenizer {
/**
* Returns the last unambiguous break position in the text.
- *
+ *
* @return position of character, or -1 if one does not exist
*/
private int findSafeEnd() {
- for (int i = length - 1; i >= 0; i--)
- if (UCharacter.isWhitespace(buffer[i]))
+ for (int i = length - 1; i >= 0; i--) {
+ if (UCharacter.isWhitespace(buffer[i])) {
return i + 1;
+ }
+ }
return -1;
}
/**
- * Refill the buffer, accumulating the offset and setting usableLength to the
- * last unambiguous break position
- *
+ * Refill the buffer, accumulating the offset and setting usableLength to the last unambiguous
+ * break position
+ *
* @throws IOException If there is a low-level I/O error.
*/
private void refill() throws IOException {
@@ -162,15 +159,19 @@ public final class ICUTokenizer extends Tokenizer {
int requested = buffer.length - leftover;
int returned = read(input, buffer, leftover, requested);
length = returned + leftover;
- if (returned < requested) /* reader has been emptied, process the rest */
+ if (returned < requested) {
+ /* reader has been emptied, process the rest */
usableLength = length;
- else { /* still more data to be read, find a safe-stopping place */
+ } else {
+ /* still more data to be read, find a safe-stopping place */
usableLength = findSafeEnd();
- if (usableLength < 0)
- usableLength = length; /*
- * more than IOBUFFER of text without space,
- * gonna possibly truncate tokens
- */
+ if (usableLength < 0) {
+ /*
+ * more than IOBUFFER of text without space,
+ * gonna possibly truncate tokens
+ */
+ usableLength = length;
+ }
}
breaker.setText(buffer, 0, Math.max(0, usableLength));
@@ -181,12 +182,12 @@ public final class ICUTokenizer extends Tokenizer {
/** commons-io's readFully, but without bugs if offset != 0 */
private static int read(Reader input, char[] buffer, int offset, int length) throws IOException {
assert length >= 0 : "length must not be negative: " + length;
-
+
int remaining = length;
- while ( remaining > 0 ) {
+ while (remaining > 0) {
int location = length - remaining;
- int count = input.read( buffer, offset + location, remaining );
- if ( -1 == count ) { // EOF
+ int count = input.read(buffer, offset + location, remaining);
+ if (-1 == count) { // EOF
break;
}
remaining -= count;
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java
index e2d3dae..615f528 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java
@@ -19,23 +19,19 @@ package org.apache.lucene.analysis.icu.segmentation;
import com.ibm.icu.text.RuleBasedBreakIterator;
/**
- * Class that allows for tailored Unicode Text Segmentation on
- * a per-writing system basis.
+ * Class that allows for tailored Unicode Text Segmentation on a per-writing system basis.
+ *
* @lucene.experimental
*/
public abstract class ICUTokenizerConfig {
/** Rule status for emoji sequences */
public static final int EMOJI_SEQUENCE_STATUS = 299;
- /**
- * Sole constructor. (For invocation by subclass
- * constructors, typically implicit.)
- */
+ /** Sole constructor. (For invocation by subclass constructors, typically implicit.) */
public ICUTokenizerConfig() {}
/** Return a breakiterator capable of processing a given script. */
public abstract RuleBasedBreakIterator getBreakIterator(int script);
- /** Return a token type value for a given script and BreakIterator
- * rule status. */
+ /** Return a token type value for a given script and BreakIterator rule status. */
public abstract String getType(int script, int ruleStatus);
/** true if Han, Hiragana, and Katakana scripts should all be returned as Japanese */
public abstract boolean combineCJ();
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
index fa715c9..bd79767 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
@@ -16,7 +16,10 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.lang.UProperty;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
@@ -24,25 +27,17 @@ import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import org.apache.lucene.util.ResourceLoader;
-import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IOUtils;
-
-import com.ibm.icu.lang.UCharacter;
-import com.ibm.icu.lang.UProperty;
-import com.ibm.icu.text.BreakIterator;
-import com.ibm.icu.text.RuleBasedBreakIterator;
+import org.apache.lucene.util.ResourceLoader;
+import org.apache.lucene.util.ResourceLoaderAware;
/**
- * Factory for {@link ICUTokenizer}.
- * Words are broken across script boundaries, then segmented according to
- * the BreakIterator and typing provided by the {@link DefaultICUTokenizerConfig}.
+ * Factory for {@link ICUTokenizer}. Words are broken across script boundaries, then segmented
+ * according to the BreakIterator and typing provided by the {@link DefaultICUTokenizerConfig}.
*
- * <p>
- * To use the default set of per-script rules:
+ * <p>To use the default set of per-script rules:
*
* <pre class="prettyprint" >
* <fieldType name="text_icu" class="solr.TextField" positionIncrementGap="100">
@@ -51,19 +46,16 @@ import com.ibm.icu.text.RuleBasedBreakIterator;
* </analyzer>
* </fieldType></pre>
*
- * <p>
- * You can customize this tokenizer's behavior by specifying per-script rule files,
- * which are compiled by the ICU RuleBasedBreakIterator. See the
- * <a href="http://userguide.icu-project.org/boundaryanalysis#TOC-RBBI-Rules"
- * >ICU RuleBasedBreakIterator syntax reference</a>.
+ * <p>You can customize this tokenizer's behavior by specifying per-script rule files, which are
+ * compiled by the ICU RuleBasedBreakIterator. See the <a
+ * href="http://userguide.icu-project.org/boundaryanalysis#TOC-RBBI-Rules">ICU
+ * RuleBasedBreakIterator syntax reference</a>.
*
- * <p>
- * To add per-script rules, add a "rulefiles" argument, which should contain a
- * comma-separated list of <code>code:rulefile</code> pairs in the following format:
- * <a href="http://unicode.org/iso15924/iso15924-codes.html"
- * >four-letter ISO 15924 script code</a>, followed by a colon, then a resource
- * path. E.g. to specify rules for Latin (script code "Latn") and Cyrillic
- * (script code "Cyrl"):
+ * <p>To add per-script rules, add a "rulefiles" argument, which should contain a comma-separated
+ * list of <code>code:rulefile</code> pairs in the following format: <a
+ * href="http://unicode.org/iso15924/iso15924-codes.html" >four-letter ISO 15924 script code</a>,
+ * followed by a colon, then a resource path. E.g. to specify rules for Latin (script code "Latn")
+ * and Cyrillic (script code "Cyrl"):
*
* <pre class="prettyprint" >
* <fieldType name="text_icu_custom" class="solr.TextField" positionIncrementGap="100">
@@ -82,13 +74,13 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa
public static final String NAME = "icu";
static final String RULEFILES = "rulefiles";
- private final Map<Integer,String> tailored;
+ private final Map<Integer, String> tailored;
private ICUTokenizerConfig config;
private final boolean cjkAsWords;
private final boolean myanmarAsWords;
-
+
/** Creates a new ICUTokenizerFactory */
- public ICUTokenizerFactory(Map<String,String> args) {
+ public ICUTokenizerFactory(Map<String, String> args) {
super(args);
tailored = new HashMap<>();
String rulefilesArg = get(args, RULEFILES);
@@ -97,7 +89,7 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa
for (String scriptAndResourcePath : scriptAndResourcePaths) {
int colonPos = scriptAndResourcePath.indexOf(":");
String scriptCode = scriptAndResourcePath.substring(0, colonPos).trim();
- String resourcePath = scriptAndResourcePath.substring(colonPos+1).trim();
+ String resourcePath = scriptAndResourcePath.substring(colonPos + 1).trim();
tailored.put(UCharacter.getPropertyValueEnum(UProperty.SCRIPT, scriptCode), resourcePath);
}
}
@@ -119,36 +111,39 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa
if (tailored.isEmpty()) {
config = new DefaultICUTokenizerConfig(cjkAsWords, myanmarAsWords);
} else {
- final BreakIterator breakers[] = new BreakIterator[1 + UCharacter.getIntPropertyMaxValue(UProperty.SCRIPT)];
- for (Map.Entry<Integer,String> entry : tailored.entrySet()) {
+ final BreakIterator breakers[] =
+ new BreakIterator[1 + UCharacter.getIntPropertyMaxValue(UProperty.SCRIPT)];
+ for (Map.Entry<Integer, String> entry : tailored.entrySet()) {
int code = entry.getKey();
String resourcePath = entry.getValue();
breakers[code] = parseRules(resourcePath, loader);
}
- config = new DefaultICUTokenizerConfig(cjkAsWords, myanmarAsWords) {
-
- @Override
- public RuleBasedBreakIterator getBreakIterator(int script) {
- if (breakers[script] != null) {
- return (RuleBasedBreakIterator) breakers[script].clone();
- } else {
- return super.getBreakIterator(script);
- }
- }
- // TODO: we could also allow codes->types mapping
- };
+ config =
+ new DefaultICUTokenizerConfig(cjkAsWords, myanmarAsWords) {
+
+ @Override
+ public RuleBasedBreakIterator getBreakIterator(int script) {
+ if (breakers[script] != null) {
+ return (RuleBasedBreakIterator) breakers[script].clone();
+ } else {
+ return super.getBreakIterator(script);
+ }
+ }
+ // TODO: we could also allow codes->types mapping
+ };
}
}
-
+
private BreakIterator parseRules(String filename, ResourceLoader loader) throws IOException {
StringBuilder rules = new StringBuilder();
InputStream rulesStream = loader.openResource(filename);
- BufferedReader reader = new BufferedReader
- (IOUtils.getDecodingReader(rulesStream, StandardCharsets.UTF_8));
+ BufferedReader reader =
+ new BufferedReader(IOUtils.getDecodingReader(rulesStream, StandardCharsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
- if ( ! line.startsWith("#"))
+ if (!line.startsWith("#")) {
rules.append(line);
+ }
rules.append('\n');
}
reader.close();
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java
index 4dd723e..4cbd0d5 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java
@@ -2,27 +2,27 @@
* Copyright (C) 1999-2010, International Business Machines
* Corporation and others. All Rights Reserved.
*
- * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
- * Software, and to permit persons to whom the Software is furnished to do so,
- * provided that the above copyright notice(s) and this permission notice appear
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * provided that the above copyright notice(s) and this permission notice appear
* in all copies of the Software and that both the above copyright notice(s) and
* this permission notice appear in supporting documentation.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
- * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE
- * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR
- * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
- * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE
+ * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
+ * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
- * Except as contained in this notice, the name of a copyright holder shall not
- * be used in advertising or otherwise to promote the sale, use or other
- * dealings in this Software without prior written authorization of the
+ * Except as contained in this notice, the name of a copyright holder shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization of the
* copyright holder.
*/
package org.apache.lucene.analysis.icu.segmentation;
@@ -33,20 +33,21 @@ import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.UTF16;
/**
- * An iterator that locates ISO 15924 script boundaries in text.
- * <p>
- * This is not the same as simply looking at the Unicode block, or even the
- * Script property. Some characters are 'common' across multiple scripts, and
- * some 'inherit' the script value of text surrounding them.
- * <p>
- * This is similar to ICU (internal-only) UScriptRun, with the following
- * differences:
+ * An iterator that locates ISO 15924 script boundaries in text.
+ *
+ * <p>This is not the same as simply looking at the Unicode block, or even the Script property. Some
+ * characters are 'common' across multiple scripts, and some 'inherit' the script value of text
+ * surrounding them.
+ *
+ * <p>This is similar to ICU (internal-only) UScriptRun, with the following differences:
+ *
* <ul>
- * <li>Doesn't attempt to match paired punctuation. For tokenization purposes, this
- * is not necessary. It's also quite expensive.
- * <li>Non-spacing marks inherit the script of their base character, following
- * recommendations from UTR #24.
+ * <li>Doesn't attempt to match paired punctuation. For tokenization purposes, this is not
+ * necessary. It's also quite expensive.
+ * <li>Non-spacing marks inherit the script of their base character, following recommendations
+ * from UTR #24.
* </ul>
+ *
* @lucene.experimental
*/
final class ScriptIterator {
@@ -58,19 +59,17 @@ final class ScriptIterator {
private int scriptStart;
private int scriptLimit;
private int scriptCode;
-
+
private final boolean combineCJ;
-
- /**
- * @param combineCJ if true: Han,Hiragana,Katakana will all return as {@link UScript#JAPANESE}
- */
+
+ /** @param combineCJ if true: Han,Hiragana,Katakana will all return as {@link UScript#JAPANESE} */
ScriptIterator(boolean combineCJ) {
this.combineCJ = combineCJ;
}
/**
* Get the start of this script run
- *
+ *
* @return start position of script run
*/
int getScriptStart() {
@@ -79,7 +78,7 @@ final class ScriptIterator {
/**
* Get the index of the first character after the end of this script run
- *
+ *
* @return position of the first character after this script run
*/
int getScriptLimit() {
@@ -88,7 +87,7 @@ final class ScriptIterator {
/**
* Get the UScript script code for this script run
- *
+ *
* @return code for the script of the current run
*/
int getScriptCode() {
@@ -97,12 +96,11 @@ final class ScriptIterator {
/**
* Iterates to the next script run, returning true if one exists.
- *
+ *
* @return true if there is another script run, false otherwise.
*/
boolean next() {
- if (scriptLimit >= limit)
- return false;
+ if (scriptLimit >= limit) return false;
scriptCode = UScript.COMMON;
scriptStart = scriptLimit;
@@ -140,13 +138,14 @@ final class ScriptIterator {
/** Determine if two scripts are compatible. */
private static boolean isSameScript(int scriptOne, int scriptTwo) {
- return scriptOne <= UScript.INHERITED || scriptTwo <= UScript.INHERITED
+ return scriptOne <= UScript.INHERITED
+ || scriptTwo <= UScript.INHERITED
|| scriptOne == scriptTwo;
}
/**
* Set a new region of text to be examined by this iterator
- *
+ *
* @param text text buffer to examine
* @param start offset into buffer
* @param length maximum length to examine
@@ -165,8 +164,9 @@ final class ScriptIterator {
private static final int basicLatin[] = new int[128];
static {
- for (int i = 0; i < basicLatin.length; i++)
+ for (int i = 0; i < basicLatin.length; i++) {
basicLatin[i] = UScript.getScript(i);
+ }
}
/** fast version of UScript.getScript(). Basic Latin is an array lookup */
@@ -181,7 +181,7 @@ final class ScriptIterator {
} else if (codepoint >= 0xFF10 && codepoint <= 0xFF19) {
// when using CJK dictionary breaking, don't let full width numbers go to it, otherwise
// they are treated as punctuation. we currently have no cleaner way to fix this!
- return UScript.LATIN;
+ return UScript.LATIN;
} else {
return script;
}
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package-info.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package-info.java
index 81c904f..3757831 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package-info.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Tokenizer that breaks text into words with the Unicode Text Segmentation algorithm.
- */
+/** Tokenizer that breaks text into words with the Unicode Text Segmentation algorithm. */
package org.apache.lucene.analysis.icu.segmentation;
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ICUCollatedTermAttributeImpl.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ICUCollatedTermAttributeImpl.java
index 0e5c98e..1f02df8 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ICUCollatedTermAttributeImpl.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ICUCollatedTermAttributeImpl.java
@@ -16,16 +16,14 @@
*/
package org.apache.lucene.analysis.icu.tokenattributes;
-
-import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
-import org.apache.lucene.util.BytesRef;
-
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
+import org.apache.lucene.util.BytesRef;
/**
- * Extension of {@link CharTermAttributeImpl} that encodes the term
- * text as a binary Unicode collation key instead of as UTF-8 bytes.
+ * Extension of {@link CharTermAttributeImpl} that encodes the term text as a binary Unicode
+ * collation key instead of as UTF-8 bytes.
*/
public class ICUCollatedTermAttributeImpl extends CharTermAttributeImpl {
private final Collator collator;
@@ -33,6 +31,7 @@ public class ICUCollatedTermAttributeImpl extends CharTermAttributeImpl {
/**
* Create a new ICUCollatedTermAttributeImpl
+ *
* @param collator Collation key generator
*/
public ICUCollatedTermAttributeImpl(Collator collator) {
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttribute.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttribute.java
index 54c8341..73ce746 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttribute.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttribute.java
@@ -16,35 +16,36 @@
*/
package org.apache.lucene.analysis.icu.tokenattributes;
-
-import org.apache.lucene.util.Attribute;
-
import com.ibm.icu.lang.UScript; // javadoc @link
+import org.apache.lucene.util.Attribute;
/**
* This attribute stores the UTR #24 script value for a token of text.
+ *
* @lucene.experimental
*/
public interface ScriptAttribute extends Attribute {
/**
- * Get the numeric code for this script value.
- * This is the constant value from {@link UScript}.
+ * Get the numeric code for this script value. This is the constant value from {@link UScript}.
+ *
* @return numeric code
*/
public int getCode();
/**
- * Set the numeric code for this script value.
- * This is the constant value from {@link UScript}.
+ * Set the numeric code for this script value. This is the constant value from {@link UScript}.
+ *
* @param code numeric code
*/
public void setCode(int code);
/**
* Get the full name.
+ *
* @return UTR #24 full name.
*/
public String getName();
/**
* Get the abbreviated name.
+ *
* @return UTR #24 abbreviated name.
*/
public String getShortName();
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
index ed12fbd..d2a389b 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
@@ -16,28 +16,26 @@
*/
package org.apache.lucene.analysis.icu.tokenattributes;
-
+import com.ibm.icu.lang.UScript;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
-import com.ibm.icu.lang.UScript;
-
/**
- * Implementation of {@link ScriptAttribute} that stores the script
- * as an integer.
+ * Implementation of {@link ScriptAttribute} that stores the script as an integer.
+ *
* @lucene.experimental
*/
public class ScriptAttributeImpl extends AttributeImpl implements ScriptAttribute, Cloneable {
private int code = UScript.COMMON;
-
+
/** Initializes this attribute with <code>UScript.COMMON</code> */
public ScriptAttributeImpl() {}
-
+
@Override
public int getCode() {
return code;
}
-
+
@Override
public void setCode(int code) {
this.code = code;
@@ -52,7 +50,7 @@ public class ScriptAttributeImpl extends AttributeImpl implements ScriptAttribut
public String getShortName() {
return UScript.getShortName(code);
}
-
+
@Override
public void clear() {
code = UScript.COMMON;
@@ -63,17 +61,17 @@ public class ScriptAttributeImpl extends AttributeImpl implements ScriptAttribut
ScriptAttribute t = (ScriptAttribute) target;
t.setCode(code);
}
-
+
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
-
+
if (other instanceof ScriptAttributeImpl) {
return ((ScriptAttributeImpl) other).code == code;
}
-
+
return false;
}
@@ -84,8 +82,8 @@ public class ScriptAttributeImpl extends AttributeImpl implements ScriptAttribut
@Override
public void reflectWith(AttributeReflector reflector) {
- // when wordbreaking CJK, we use the 15924 code Japanese (Han+Hiragana+Katakana) to
- // mark runs of Chinese/Japanese. our use is correct (as for chinese Han is a subset),
+ // when wordbreaking CJK, we use the 15924 code Japanese (Han+Hiragana+Katakana) to
+ // mark runs of Chinese/Japanese. our use is correct (as for chinese Han is a subset),
// but this is just to help prevent confusion.
String name = code == UScript.JAPANESE ? "Chinese/Japanese" : getName();
reflector.reflect(ScriptAttribute.class, "script", name);
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/package-info.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/package-info.java
index d600eb3..bbd1021 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/package-info.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Additional ICU-specific Attributes for text analysis.
- */
+/** Additional ICU-specific Attributes for text analysis. */
package org.apache.lucene.analysis.icu.tokenattributes;
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationDocValuesField.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationDocValuesField.java
index 4e90a48..a53a928 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationDocValuesField.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationDocValuesField.java
@@ -16,7 +16,8 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.util.ULocale;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
@@ -34,45 +35,41 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import com.ibm.icu.text.Collator;
-import com.ibm.icu.util.ULocale;
-
-/**
- * trivial test of ICUCollationDocValuesField
- */
+/** trivial test of ICUCollationDocValuesField */
public class TestICUCollationDocValuesField extends LuceneTestCase {
-
+
public void testBasic() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
Document doc = new Document();
Field field = newField("field", "", StringField.TYPE_STORED);
- ICUCollationDocValuesField collationField = new ICUCollationDocValuesField("collated", Collator.getInstance(ULocale.ENGLISH));
+ ICUCollationDocValuesField collationField =
+ new ICUCollationDocValuesField("collated", Collator.getInstance(ULocale.ENGLISH));
doc.add(field);
doc.add(collationField);
field.setStringValue("ABC");
collationField.setStringValue("ABC");
iw.addDocument(doc);
-
+
field.setStringValue("abc");
collationField.setStringValue("abc");
iw.addDocument(doc);
-
+
IndexReader ir = iw.getReader();
iw.close();
-
+
IndexSearcher is = newSearcher(ir);
-
+
SortField sortField = new SortField("collated", SortField.Type.STRING);
-
+
TopDocs td = is.search(new MatchAllDocsQuery(), 5, new Sort(sortField));
assertEquals("abc", ir.document(td.scoreDocs[0].doc).get("field"));
assertEquals("ABC", ir.document(td.scoreDocs[1].doc).get("field"));
ir.close();
dir.close();
}
-
+
public void testRanges() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
@@ -82,10 +79,11 @@ public class TestICUCollationDocValuesField extends LuceneTestCase {
if (random().nextBoolean()) {
collator.setStrength(Collator.PRIMARY);
}
- ICUCollationDocValuesField collationField = new ICUCollationDocValuesField("collated", collator);
+ ICUCollationDocValuesField collationField =
+ new ICUCollationDocValuesField("collated", collator);
doc.add(field);
doc.add(collationField);
-
+
int numDocs = atLeast(500);
for (int i = 0; i < numDocs; i++) {
String value = TestUtil.randomSimpleString(random());
@@ -93,11 +91,11 @@ public class TestICUCollationDocValuesField extends LuceneTestCase {
collationField.setStringValue(value);
iw.addDocument(doc);
}
-
+
IndexReader ir = iw.getReader();
iw.close();
IndexSearcher is = newSearcher(ir);
-
+
int numChecks = atLeast(100);
for (int i = 0; i < numChecks; i++) {
String start = TestUtil.randomSimpleString(random());
@@ -106,17 +104,25 @@ public class TestICUCollationDocValuesField extends LuceneTestCase {
BytesRef upperVal = new BytesRef(collator.getCollationKey(end).toByteArray());
doTestRanges(is, start, end, lowerVal, upperVal, collator);
}
-
+
ir.close();
dir.close();
}
-
- private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception {
+
+ private void doTestRanges(
+ IndexSearcher is,
+ String startPoint,
+ String endPoint,
+ BytesRef startBR,
+ BytesRef endBR,
+ Collator collator)
+ throws Exception {
SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
- for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
+ for (int docID = 0; docID < is.getIndexReader().maxDoc(); docID++) {
Document doc = is.doc(docID);
String s = doc.getField("field").stringValue();
- boolean collatorAccepts = collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
+ boolean collatorAccepts =
+ collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
assertEquals(docID, dvs.nextDoc());
BytesRef br = dvs.binaryValue();
boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationKeyAnalyzer.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationKeyAnalyzer.java
index a85d583..877d34a 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationKeyAnalyzer.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUCollationKeyAnalyzer.java
@@ -16,55 +16,51 @@
*/
package org.apache.lucene.analysis.icu;
-
import com.ibm.icu.text.Collator;
-
+import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CollationTestBase;
import org.apache.lucene.util.BytesRef;
-import java.util.Locale;
-
public class TestICUCollationKeyAnalyzer extends CollationTestBase {
private Collator collator = Collator.getInstance(new Locale("fa"));
private Analyzer analyzer;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
analyzer = new ICUCollationKeyAnalyzer(collator);
}
-
+
@Override
public void tearDown() throws Exception {
analyzer.close();
super.tearDown();
}
- private BytesRef firstRangeBeginning = new BytesRef
- (collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());
- private BytesRef firstRangeEnd = new BytesRef
- (collator.getCollationKey(firstRangeEndOriginal).toByteArray());
- private BytesRef secondRangeBeginning = new BytesRef
- (collator.getCollationKey(secondRangeBeginningOriginal).toByteArray());
- private BytesRef secondRangeEnd = new BytesRef
- (collator.getCollationKey(secondRangeEndOriginal).toByteArray());
+ private BytesRef firstRangeBeginning =
+ new BytesRef(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());
+ private BytesRef firstRangeEnd =
+ new BytesRef(collator.getCollationKey(firstRangeEndOriginal).toByteArray());
+ private BytesRef secondRangeBeginning =
+ new BytesRef(collator.getCollationKey(secondRangeBeginningOriginal).toByteArray());
+ private BytesRef secondRangeEnd =
+ new BytesRef(collator.getCollationKey(secondRangeEndOriginal).toByteArray());
public void testFarsiRangeFilterCollating() throws Exception {
- testFarsiRangeFilterCollating(analyzer, firstRangeBeginning, firstRangeEnd,
- secondRangeBeginning, secondRangeEnd);
+ testFarsiRangeFilterCollating(
+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);
}
-
+
public void testFarsiRangeQueryCollating() throws Exception {
- testFarsiRangeQueryCollating(analyzer, firstRangeBeginning, firstRangeEnd,
- secondRangeBeginning, secondRangeEnd);
+ testFarsiRangeQueryCollating(
+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);
}
public void testFarsiTermRangeQuery() throws Exception {
- testFarsiTermRangeQuery
- (analyzer, firstRangeBeginning, firstRangeEnd,
- secondRangeBeginning, secondRangeEnd);
+ testFarsiTermRangeQuery(
+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);
}
public void testThreadSafe() throws Exception {
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
index b668db8..67d065c 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
@@ -16,92 +16,90 @@
*/
package org.apache.lucene.analysis.icu;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-/**
- * Tests ICUFoldingFilter
- */
+/** Tests ICUFoldingFilter */
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
Analyzer a;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
- a = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
- }
- };
+ a =
+ new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
+ }
+ };
}
-
+
@Override
public void tearDown() throws Exception {
a.close();
super.tearDown();
}
-
+
public void testDefaults() throws IOException {
// case folding
- assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" });
+ assertAnalyzesTo(a, "This is a test", new String[] {"this", "is", "a", "test"});
// case folding
- assertAnalyzesTo(a, "Ruß", new String[] { "russ" });
-
+ assertAnalyzesTo(a, "Ruß", new String[] {"russ"});
+
// case folding with accent removal
- assertAnalyzesTo(a, "ΜΆΪΟΣ", new String[] { "μαιοσ" });
- assertAnalyzesTo(a, "Μάϊος", new String[] { "μαιοσ" });
+ assertAnalyzesTo(a, "ΜΆΪΟΣ", new String[] {"μαιοσ"});
+ assertAnalyzesTo(a, "Μάϊος", new String[] {"μαιοσ"});
// supplementary case folding
- assertAnalyzesTo(a, "𐐖", new String[] { "𐐾" });
-
+ assertAnalyzesTo(a, "𐐖", new String[] {"𐐾"});
+
// normalization
- assertAnalyzesTo(a, "ﴳﴺﰧ", new String[] { "طمطمطم" });
+ assertAnalyzesTo(a, "ﴳﴺﰧ", new String[] {"طمطمطم"});
// removal of default ignorables
- assertAnalyzesTo(a, "क्ष", new String[] { "कष" });
-
+ assertAnalyzesTo(a, "क्ष", new String[] {"कष"});
+
// removal of latin accents (composed)
- assertAnalyzesTo(a, "résumé", new String[] { "resume" });
-
+ assertAnalyzesTo(a, "résumé", new String[] {"resume"});
+
// removal of latin accents (decomposed)
- assertAnalyzesTo(a, "re\u0301sume\u0301", new String[] { "resume" });
-
+ assertAnalyzesTo(a, "re\u0301sume\u0301", new String[] {"resume"});
+
// fold native digits
- assertAnalyzesTo(a, "৭০৬", new String[] { "706" });
-
+ assertAnalyzesTo(a, "৭০৬", new String[] {"706"});
+
// ascii-folding-filter type stuff
- assertAnalyzesTo(a, "đis is cræzy", new String[] { "dis", "is", "craezy" });
+ assertAnalyzesTo(a, "đis is cræzy", new String[] {"dis", "is", "craezy"});
// proper downcasing of Turkish dotted-capital I
// (according to default case folding rules)
- assertAnalyzesTo(a, "ELİF", new String[] { "elif" });
-
+ assertAnalyzesTo(a, "ELİF", new String[] {"elif"});
+
// handling of decomposed combining-dot-above
- assertAnalyzesTo(a, "eli\u0307f", new String[] { "elif" });
+ assertAnalyzesTo(a, "eli\u0307f", new String[] {"elif"});
}
-
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
}
-
+
public void testEmptyTerm() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
- return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
+ }
+ };
checkOneTerm(a, "", "");
a.close();
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java
index 3e3c523..bf19a4d 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java
@@ -16,11 +16,9 @@
*/
package org.apache.lucene.analysis.icu;
-
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
@@ -30,36 +28,42 @@ public class TestICUFoldingFilterFactory extends BaseTokenStreamTestCase {
/** basic tests to ensure the folding is working */
public void test() throws Exception {
Reader reader = new StringReader("Résumé");
- ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory(new HashMap<String,String>());
+ ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory(new HashMap<String, String>());
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "resume" });
+ assertTokenStreamContents(stream, new String[] {"resume"});
}
/** test to ensure the filter parameter is working */
public void testFilter() throws Exception {
- HashMap<String,String> args = new HashMap<String,String>();
+ HashMap<String, String> args = new HashMap<String, String>();
args.put("filter", "[^ö]");
ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory(args);
Reader reader = new StringReader("Résumé");
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "resume" });
+ assertTokenStreamContents(stream, new String[] {"resume"});
reader = new StringReader("Fönster");
stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "fönster" });
+ assertTokenStreamContents(stream, new String[] {"fönster"});
}
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new ICUFoldingFilterFactory(new HashMap<String,String>() {{
- put("bogusArg", "bogusValue");
- }});
- });
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ new ICUFoldingFilterFactory(
+ new HashMap<String, String>() {
+ {
+ put("bogusArg", "bogusValue");
+ }
+ });
+ });
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
index c090d07..bb1240c 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
@@ -16,12 +16,11 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Normalizer2;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Arrays;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharFilter;
@@ -31,8 +30,6 @@ import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.ngram.NGramTokenizer;
import org.apache.lucene.util.TestUtil;
-import com.ibm.icu.text.Normalizer2;
-
public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
public void testNormalization() throws IOException {
@@ -49,7 +46,9 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
break;
}
output.append(tempBuff, 0, length);
- assertEquals(output.toString(), normalizer.normalize(input.substring(0, reader.correctOffset(output.length()))));
+ assertEquals(
+ output.toString(),
+ normalizer.normalize(input.substring(0, reader.correctOffset(output.length()))));
}
assertEquals(expectedOutput, output.toString());
@@ -59,66 +58,76 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
// '℃', '№', '㈱', '㌘', 'ｻ'+'<<', 'ｿ'+'<<', '㌰'+'<<'
String input = "℃ № ㈱ ㌘ ｻﾞ ｿﾞ ㌰ﾞ";
- CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
- Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE));
+ CharFilter reader =
+ new ICUNormalizer2CharFilter(
+ new StringReader(input),
+ Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE));
Tokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenStream.setReader(reader);
- assertTokenStreamContents(tokenStream,
- new String[] {"°C", "No", "(株)", "グラム", "ザ", "ゾ", "ピゴ"},
- new int[] {0, 2, 4, 6, 8, 11, 14},
- new int[] {1, 3, 5, 7, 10, 13, 16},
- input.length());
+ assertTokenStreamContents(
+ tokenStream,
+ new String[] {"°C", "No", "(株)", "グラム", "ザ", "ゾ", "ピゴ"},
+ new int[] {0, 2, 4, 6, 8, 11, 14},
+ new int[] {1, 3, 5, 7, 10, 13, 16},
+ input.length());
}
public void testTokenStream2() throws IOException {
// '㌰', '<<'゙, '5', '℃', '№', '㈱', '㌘', 'ｻ', '<<', 'ｿ', '<<'
String input = "㌰゙5℃№㈱㌘ｻﾞｿﾞ";
- CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
- Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ CharFilter reader =
+ new ICUNormalizer2CharFilter(
+ new StringReader(input),
+ Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
Tokenizer tokenStream = new NGramTokenizer(newAttributeFactory(), 1, 1);
tokenStream.setReader(reader);
- assertTokenStreamContents(tokenStream,
- new String[] {"ピ", "ゴ", "5", "°", "c", "n", "o", "(", "株", ")", "グ", "ラ", "ム", "ザ", "ゾ"},
- new int[]{0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 7, 9},
- new int[]{1, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 7, 9, 11},
- input.length()
- );
+ assertTokenStreamContents(
+ tokenStream,
+ new String[] {"ピ", "ゴ", "5", "°", "c", "n", "o", "(", "株", ")", "グ", "ラ", "ム", "ザ", "ゾ"},
+ new int[] {0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 7, 9},
+ new int[] {1, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 7, 9, 11},
+ input.length());
}
-
+
public void testMassiveLigature() throws IOException {
String input = "\uFDFA";
- CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
- Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ CharFilter reader =
+ new ICUNormalizer2CharFilter(
+ new StringReader(input),
+ Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
Tokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenStream.setReader(reader);
- assertTokenStreamContents(tokenStream,
- new String[] {"صلى", "الله", "عليه", "وسلم"},
- new int[]{0, 0, 0, 0},
- new int[]{0, 0, 0, 1},
- input.length()
- );
+ assertTokenStreamContents(
+ tokenStream,
+ new String[] {"صلى", "الله", "عليه", "وسلم"},
+ new int[] {0, 0, 0, 0},
+ new int[] {0, 0, 0, 1},
+ input.length());
}
- public void doTestMode(final Normalizer2 normalizer, int maxLength, int iterations, int bufferSize) throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.KEYWORD, false));
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, normalizer, bufferSize);
- }
- };
+ public void doTestMode(
+ final Normalizer2 normalizer, int maxLength, int iterations, int bufferSize)
+ throws IOException {
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.KEYWORD, false));
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(reader, normalizer, bufferSize);
+ }
+ };
for (int i = 0; i < iterations; i++) {
String input = TestUtil.randomUnicodeString(random(), maxLength);
@@ -135,286 +144,335 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
}
public void testNFC() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.COMPOSE), 20, RANDOM_MULTIPLIER*1000, 128);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.COMPOSE),
+ 20,
+ RANDOM_MULTIPLIER * 1000,
+ 128);
}
-
+
public void testNFCHuge() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.COMPOSE), 256, RANDOM_MULTIPLIER*500, 16);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.COMPOSE),
+ 256,
+ RANDOM_MULTIPLIER * 500,
+ 16);
}
public void testNFD() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE), 20, RANDOM_MULTIPLIER*1000, 128);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE),
+ 20,
+ RANDOM_MULTIPLIER * 1000,
+ 128);
}
-
+
public void testNFDHuge() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE), 256, RANDOM_MULTIPLIER*500, 16);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE),
+ 256,
+ RANDOM_MULTIPLIER * 500,
+ 16);
}
public void testNFKC() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE), 20, RANDOM_MULTIPLIER*1000, 128);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE),
+ 20,
+ RANDOM_MULTIPLIER * 1000,
+ 128);
}
-
+
public void testNFKCHuge() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE), 256, RANDOM_MULTIPLIER*500, 16);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.COMPOSE),
+ 256,
+ RANDOM_MULTIPLIER * 500,
+ 16);
}
public void testNFKD() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE), 20, RANDOM_MULTIPLIER*1000, 128);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE),
+ 20,
+ RANDOM_MULTIPLIER * 1000,
+ 128);
}
-
+
public void testNFKDHuge() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE), 256, RANDOM_MULTIPLIER*500, 16);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE),
+ 256,
+ RANDOM_MULTIPLIER * 500,
+ 16);
}
public void testNFKC_CF() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE), 20, RANDOM_MULTIPLIER*1000, 128);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE),
+ 20,
+ RANDOM_MULTIPLIER * 1000,
+ 128);
}
-
+
public void testNFKC_CFHuge() throws Exception {
- doTestMode(Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE), 256, RANDOM_MULTIPLIER*500, 16);
+ doTestMode(
+ Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE),
+ 256,
+ RANDOM_MULTIPLIER * 500,
+ 16);
}
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-5595")
public void testRandomStrings() throws IOException {
// nfkc_cf
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(
+ reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ }
+ };
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
// huge strings
- checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192);
+ checkRandomData(random(), a, 25 * RANDOM_MULTIPLIER, 8192);
a.close();
// nfkd
- a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE));
- }
- };
+ a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(
+ reader, Normalizer2.getInstance(null, "nfkc", Normalizer2.Mode.DECOMPOSE));
+ }
+ };
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
// huge strings
- checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192);
+ checkRandomData(random(), a, 25 * RANDOM_MULTIPLIER, 8192);
a.close();
}
-
+
public void testCuriousString() throws Exception {
String text = "\udb40\udc3d\uf273\ue960\u06c8\ud955\udc13\ub7fc\u0692 \u2089\u207b\u2073\u2075";
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(
+ reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ }
+ };
for (int i = 0; i < 1000; i++) {
checkAnalysisConsistency(random(), a, false, text);
}
a.close();
}
-
- public void testCuriousMassiveString() throws Exception {
- String text = "yqt \u0728\u0707\u0712\u0720\u0734 \u204c\u201d hyipy \u2667\u2619" +
- "\u26ec\u267b\u26da uboyjwfbv \u2012\u205d\u2042\u200a\u2047\u2040 gyxmmz yvv %" +
- "\ufb86 \n<script> hupvobbv jvsztd x rww ct{1,5} brteyi dfgyzqbm hdykd ahgeizyhv" +
- " kLn c#\ud8f8\udd74 fPezd ktedq \ufcea=\ud997\uddc9\u876bJ\u0231\ud98a\udce0\uf872" +
- " zquqah \ub9d6\u144e\uc686 3\ud93d\udfca\u1215\ud614 tnorask \u0348\u0334\u0334" +
- "\u0300\u033d geqdeoghh foojebut \ufb52\u227ag\ud9bd\udc3a\u07efK nyantr lksxw fs" +
- "zies ubzqfolksjpgk \u6fa3\ud859\udc36\u0501\ucca0\u0306\u001e\ua756\u052f \ucaf7" +
- "\u0247\u0009\ufdddg \ud83c\udd02\ud83c\uddaf \u05628\u2b49\u02e3\u0718\u0769\u4f1b" +
- "\u0496\u0766\uecaa\ufb44 \u001d \u0006hr\u00f0\ue649\u041a\uda6f\udfa5\uf31b\ue274" +
- " ptgjf \ud8cc\udf83M\u0013\u04c6i \u205f\u2004\u2032\u2001\u2057\u2066 \u07d0\uacdb" +
- "\u06a5z pqfxwgbwe \ud1bc\u2eba\u2d45\u02ee\u56df xnujtfs \u1b19\u1b17\u1b39\u1b20" +
- "\u1b69\u1b58\u1b03\u1b6e\u1b73\u1b20 afsl zxlzziqh ahrhckhktf \ud801\udc5b\ud801\udc61" +
- " bkpmeyyqobwi qnkunmpjpihezll plhhws \u37f7\u41d6\u3dca\u3e80\u4923\u36b8 \u195a\u1959" +
- "\u196f\u1968\u1958\u197f </p \u0006s\u019f\uda82\udc90H_\u079d\ufd6f: idpp \u217c" +
- "\u2168\u2185\u2179\u2156\u2159\u217c\u2154\u2158 ({1,5}( jkieylqzmb bfirnaj \uea71" +
- "\uf17f\u0749\u054c \ud9ea\udf83\udbea\ude91j x\u3366\u09c2\ud828\udd13~\u6fda\ueeee" +
- " \ud834\udcd3\ud834\udc2b\ud834\udc8b\ud834\udcd8 dl \ud802\ude3a\ud802\ude36\ud802" +
- "\ude23\ud802\ude56 \u20ad\u20a0\u20a0\u20a0\u20b8\u20b4\u20ad lcql \u0547\u0156]" +
- "\ue344V] \ud90c\udd7d\u000eS\uf965\uf15e\u008f qdn \udac3\ude3c buy m qni \u31a4\u31a3" +
- "\u31b2\u31a9\u31a7 \u20df\u20ee\u20d3\u20d4\u20f1\u20de \u4dc1\u4dc4\u4dde\u4dec\u4df9" +
- "\u4dee\u4dc5 \udb40\udc36 gstfqnfWY \u24ab\u8d69\u0301 n?]v||]i )- \udb0b\ude77\uf634" +
- "\u0762 tpkkjlbcntsk eebtzirw xo hktxy \n</ vxro xpr mtlp p|tjf|i?.- lxpfo \udbd7" +
- "\udf78\udbf5\udf57 u..b jjj]p?e jtckvhqb \u20eb\u20ea\u20fa\u20ef\u20e1\u20ed\u20eb vvm" +
- "uhbsvyi jx mqwxckf \u00d7 qqzs \u05ae\u022d\udb7c\udfb1\u070b vnhkz egnutyuat \udaa2" +
- "\udf20\ufa45#O\u2b61\u0d0e \u09a2\u0996\u09cd\u09b4 v \udbdb\ude9bqp owsln \ua837\ua833" +
- "\ua83f\ua83f\ua83f\ua831\ua83c\ua839 \u1a15\u1a1c\u1a12 \ud83c\ude20 >&pt&#x>129 \ud9f1" +
- "\udf8c\uecdd \ud809\udc48\ud809\udc72 wswskop \ua70d\ua70e ohcjsboriux \u2025\u2000\u206e" +
- "\u2039\u2036\u2002\u200e \u1741\u1741\u175e bsykot \u0696\ufab5W ifj cnosdrAxa qtv wvrdn" +
- " dmt (){0 udqg Z5U\ufcab\ue9ed\u0378 ts zndsksqtu uxbkAbn \u0535\"\u001b\ud923\udcc3\udae7" +
- "\udccf\u02ac \u0131\ua72e\u1273\u02f8\u2329\u5d83Q icakme oojxv hpec xndjzxre )e{0,5}. " +
- "\u0005!\u0009\u0004\u000bt\u0006N vqxnokp cdshqof \ua654\ua681\ua667\ua676\ua698 vwwp " +
- "\ufe4b\ufe3b\ufe4d\ufe42\ufe44\ufe38\ufe4b\ufe4c\ufe4b\ufe38 \u2ffb\u2ffa\u2ff6\u2ff7" +
- "\u2ffc\u2ff6\u2ff5 \u3116\u312c\u3100\u3102\u310f\u3116 agyueiij \u0764\u1178\ud866\udca1" +
- "\u00f2\u039d \ud802\udc12\ud802\udc1e\ud802\udc28\ud802\udc19 kygiv fxctjyj \n" +
- " omu \ud855\udeb1\u063c\ufd54\u9dbf\uf80a\ufc60 \u76ee\u3365\uf998\u70a8\u02d2 \u05e5" +
- "\u0478\u61bb \ua71c\ua708\ua70d\ua71a\ua712\ua712\ua719\ua706\ua71f\ua71c\ua71e\ua718 pgq" +
- "arvtzcoduk pyok \u1779\u1766\u1778\u177b \u16e5\u16a7\u16f3\u16fe\u16c8\u16ba\u16a4 \u0527" +
- "\u052f\u0513\u0500 iisgh hxd \u13dd\u13c6\u13db\u13ee\u13d7 \u0019\ufed4 \ud802\udd3c\ud802" +
- "\udd30\ud802\udd3d\ud802\udd24\ud802\udd2c jm gvylt eax xsbt mvuvnw \u0246\u5514\udb16\uddcf" +
- "\u1dc2\ud07b\u07af\u12e8\u8e8f\u0004 phy haduzzw \u04a1\u8334\u14b5\uf0de\udb4b\udec0\u6b69 " +
- "dubdl \u0cd2\u06c7\uf0297\u45efy-\u05e9\u01a3\uf46f aiafsh &# \u0003\ue5ab\uedcd] xhz vil" +
- "wdlytsj \uda10\udf4f\u87b2 tomuca \u1fe4\u1f4c\u1fab \u035d\u0332 xgeel nzp -)r)]r([{ nbc " +
- "\u01b4\ud80f\udff5\u0008\u0091 tbugdgv \ud9cc\udd57\u0776t\uda0f\udc17 rsuwxqxm \u2d71\u2d3d" +
- "\u2d6e zsvuicin \udb50\ude9d\u7424\u30c7\uff73\ue11e\u0014 qxtxx dlssfvxg ud \u0c1f\ud9d9" +
- "\udce4\u2317\u0482\u017b \ud83c\udd91\ud83c\uddaf\ud83c\udd84 \ud834\udf7d\ud834\udf70\ud834" +
- "\udf61 \uabfc\uabe1\uabcd\uabd1\uabe8\uabf9 \u292a\u2916\u295d\u290b\u296d\u291f \uf476\u0283" +
- "\u03d5\ucfe2 h)(o? lqeatt \u20c9\u20a5\u20cd \u1634d\u001a\ua052:\u00db \ud83c\udc8a\ud83c" +
- "\udc41\ud83c\udc8a\ud83c\udc6e k civyjm ptz \uf20d\uea32&\ud8db\ude48\uf657s\u06dc\u9aa5\udbd7" +
- "\udc74\u0111 cehuo \u2090\u209b\u2099\u208c\u209a\u2088\u2092\u207e\u207b\u2089 efpifnvdd zstz" +
- "duuyb \u04af3 \u2e5f\u2e7e\u2e7c\u2e54\u2e0f\u2e68\u2e0d\u2e05 <??> \u28d3\udbb7\udf6fJ\uf089" +
- "\ub617\ufb80\u07d0\uf141 \u0820\u083b\u0800\u0801\u0832\u0823\u0820\u081c\u0835 r laxzpfbcvz " +
- "iuwbmq scpeqaq nvw{1,5}s) \u234c\u231f\u231e\u23cc\u23d8\u2302\u2345\u231b\u239d\u231e 154614 " +
- "wgefnmgq \udbbe\udc2a\uee8c ayaeg \u243c\u2404\u2409\u241a dd hxuawoswx jqghoznw \u0019\u70cd& " +
- "\ufe0a\ufe02\ufe04\ufe0c\ufe0d \u0510\u0523\u0510\u0501\u0508\u0513\u050e ropgywv guqsrcz kmf " +
- "\u0d1f\u0d5c\u0d24\u0d5f\u0d0b\u0d14 xgklnql oprw \u0365\u034e\u036a\u0310\u034f\u0316\u031f " +
- "\u01b2\u55f6\uf1eeo\ue994\u07c4? wewz idjwsqwo \ufe67\ufe66\ufe52\ufe6a\ufe5b\ufe57\ufe68\ufe62" +
- " \u27fd\u27fe\u27f6\u27f8 fcsqqvoy edrso \u5580\ue897 vantkitto sm \uff60\ud957\udf48\uf919w" +
- "\ud889\udf3e\ud9c8\uddf6 jhas uqzmlr \ua4ce\ua4aa\ua4b3\ua4b5\ua4c2\ua4a5 kvuop ><script> " +
- "\ud802\udd0f\ud802\udd15\ud802\udd11 \u16bc gjyabb mlaewr \u1175\u11cc\u1192 ri ]\u01eb\ue4ca" +
- " \uedca\ucd66\u597c\u03df\uaf8f\u0619 &#x; ]?e \ua6c2\ua6ed\ua6eb\ua6ea\ua6cd\ua6e2 gfpafsr" +
- " pooo \u20cc\u20c4\u20a7\u20c8\u20a6\u20b0\u20ad \udb40\udd5b tgcdmeuau \u141f\u637d \ufba8" +
- "\ufcc7\ufde1\ufc85\ufbfc\ufbed b \u319a\u3193\u3192\u3195\u319e\u319d\u3195\u3194\u3197\u319c " +
- "\u0514\u0503\u051c\u050e\u0502\u0520 \u1db3\u1db5\u1d96 \ud977\udde8w\u02ec&\u01dd\u29ed\ufead" +
- "y\u03e3 ukzqq {0,5}{0 \u000f\uf028$\u046f7\udb7e\uded2 <p><l \uea5e(\uf1dcX\u376b ([mi[n(a Jev" +
- "xsJl \ubd09\u04c1\ua0f3\uff7b \u1cd3\u1cd9\u1cf6\u1cf0\u1cd8\u1cdd\u1cdd\u1ce7\u1ce3\u1cd9 " +
- "\ud800\udf91\ud800\udf99\ud800\udf83 \"135541 \u18b3\u18c0\u18c2\u18ea\u18c4\u18fe\u18b2\u18fd" +
- "\u18c3 uwykvfd lqiflow afdfctcz ol[hemp strmhxmk \ua732\ua748\ua731\ua791\ua78b\ua7ee\ua7ea" +
- "\ua789\ua766\ua7e4 gmzpmrpzr dqfp wfxwjzepdj M\udb03\udeff\u13c5 afsncdwp \ue716\u0734\ud8f9" +
- "\udc986\u0017\u0211 t r vhczf (]|q.{0 \u195e\u1973\u1958\u1956\u196c\u1973\u1966\u196c\u197a" +
- " \u2595\u258e\u259a\u2591\u2583\u2595 l kgopyql wes E\u6611\ub713\u7058^ bipq dx 7507 \ua90b" +
- "\ua90b ktjeqqx \u0e1d\u0e7f\u0e35 #(\u71b7'\u06e5\u03e4(\uf714\u6ff2 advbgh \u319c\u3191 \uef11" +
- "% \uc8a7C\ud8ed\udf4c rjb \u02ca\uf5bd\ue379n \ud834\udd7d\ud834\udd83 jracjcd rpearfd ujzvdc" +
- " ofg \u09df\u09f4\u0980\u09b3\u09bf\u09b7 \ud9cc\uddf4$\udb08\udf72 iqcnwsyjmo </scri ]-q jsit" +
- "gjg naeajiix vvmq dnlihau o cgyp tqsfe uixlzmhz sixhftpr uvtbvv mphcWojZs \u190b\ud9c3\ude7c" +
- "\u008b\u0606\ua8b1 a \u000ebq# \u1e57\u0b66\uda41\ude32\ubfd6 ohph b- ze \ue2a9\u0000 zatuye" +
- " \ufd26\ufdfa\ufbbf\ufdb4\ufde3\ufd14\ufc25\ufcb8 sbtpb nxtacgjo \ud834\ude2a\ud834\ude0f" +
- "\ud834\ude14\ud834\ude27 \ua835\ua835 ujpjkkyhujx \u001e\ud9a7\udc45\u0011>\u1aef\u0d0d <" +
- " hcefg \u01f0\u01d3 gxlwv \ud802\udd2f\ud802\udd34 \udb9c\udcc8\udbb6\ude1e\udbaf\ude33\udbae" +
- "\udc49 xyzlzynd \ud83c\udd44 vynhdwh \u315d\u3157\u314d\u3180\u317d\u318d\u317d\u3156 ?>\" " +
- "\ud800\udfdb\ud800\udfb8\ud800\udfa6\ud800\udfa7 hszn sspwldpdwjfhk vrbag \ueabd\ud9f2\udfb5" +
- "\udafb\udd28\uf6a4y\ufdeb \u0798\u078f\u0789 \ue80c\uf5c1\u001a\ud84b\uddef ywdrudo odqsts nnp" +
- "vfl nvotk rdwhr rquj cn \ud7d4\ud7b3\ud7c7\ud7bf\ud7bd � emdq pnqsbbgzs \u2c7d\u2c7e fj" +
- "kwhku >&c \ud800\udf85\ud800\udf88\ud800\udf93\ud800\udf84\ud800\udf82\ud800\udf8b '\n\"<p></p>" +
- " xltgxaa vqjmhiz n .m]c]tr( qerokel lc kugmimmtfxi <?� [g)|n|[ cij </ BwbZfg " +
- "pta bmhn \uf60dz\u54ca fwdp gnkz \u030ba\ue115}\udb7d\ude86\ud813\udc50\uedb9 \u1745\u1749\u174d" +
- "\u1747\u174b\u174f \ud802\udf09\ud802\udf3f\ud802\udf03\ud802\udf00 agmj \u1db7 \u60669\u000f" +
- "j\u000f\u02e4e\u05f5f p \udaac\udc9f\uc257\u6e58U zcwsy \u19a7\u19cf\u19aa\u199f\u19b7 zhll" +
- " sbmv .uud \u040c\u2e78\ud9fc\udd0d\ufb7f\uf2e1\u04bf vqkyxua \ud834\udf5e\ud834\udf45\ud834" +
- "\udf23 \ud834\uddbe\ud834\udd9b\ud834\uddc4 f [{0,5}t ovbsy tcj nyalqyabn yzhes mlwpw \ud835" +
- "\uddd5\ud835\udfdf\ud835\uddb4\ud835\ude3e epqfkk cssusatbbq \u1424\u1413\u1652\u15f0 dtfy zN" +
- " \u2417\u2435\u2407 qtz \u2fff\u2ff1\u2ff8\u2ff8\u2ff7\u2ff7 \ud802\ude43 gfpe stimmb ugPwlr " +
- "\u0737\u0712\u0712\u071c \u21de \u01d8\u079e\u8215\ue5b9\u07ef\u3cff\u0478 \u05dd\u05e5 gwqset" +
- "iyyh jbncv \u68ba\u52aa) kcoeiuogjsh n[nh- \uf5ff\u7ec3Z zrptjltqstzvx ts tbod 𕠗 \u07fd" +
- "\u07c1\u07c0\u07e9\u07fa\u07f2\u07e3\u07e8\u07cb\u07eb\u07d8 fisslh \ue40f\u012b\u02cf\u0766" +
- " \u1c25\u1c4f\u1c1d\u1c20 \"--> dPhwz \ud808\udef9\ud808\udf4a\ud808\uddd3 cicqhk D\ue7d3=\u5228" +
- "\udbc3\udd18\ueb0d\u0012\u0744\ufb04U\u001e\uf388c\u0306 \u2c08\u2c1e Xxzcd \u001d\u0230\u45e8" +
- "\u0653 <zlx \u1e8f\u1e28\u1e3c\u1e8d\u1ee8\u1e69 zcoeork d gusesc \ud802\udd3e nyr vkfrgi \u2cde" +
- " mo nklnsqweh <script gtoppwsbik vrcdbsx pz \udb0d\ude0c|\u93d0\uf241\u28a8\u0531 \ud83c" +
- "\udc2b\ud83c\udc10 \ud800\udf91\ud800\udf8e qwmmkty \u19f7\u19f7\u19e8\u19e0\u19f9\u19f6\u19e6" +
- " \u7a60\u7b7b\u878c\u603c\u53c6\u6552\u6dfe \u0776\u0775 foxaqi m cdjd guyqxodsccy \ucd7d\ub5eb" +
- "\ud29e\ub9ad\uba00\uac9d\ud2f0 pxjtj \ue362\u079fx\uf193 ){1,5}[{ hmpujw \u3193\u319b\u3195" +
- "\u319c\u3198\u3193\u3195\u319d\u319e \udb40\udd65\udb40\udd29\udb40\udd5c\udb40\uddba \ud801" +
- "\udc18\ud801\udc24\ud801\udc4f\ud801\udc15\ud801\udc04 \u1970\u196c\u1963\u196f\u1979 vjpeg " +
- "\ufeb9 lhoeh 吋 Szu \u0c21\u0c21\u0c36\u0c0e oyb \u1c7c\ue0ba\u001e gskf \ud826\udd47" +
- "\u0018 ooxki \u001d\u5b0d \uf0e2\u05ba\u000e\u6200 \u2ecc\u2e8a\u2eb8\u2ee5\u2edb\u2ee4\u2ec7" +
- "\u2ef9\u2e9e\u2e99 xpyijb bqvtjfjrxkjv sruyitq jobm u \u0752\u075d\u0778\u0754\u075c \ua4af" +
- "\ua4a5\ua4be\ua4a6\ua4b9\ua4b9 \ua835\ua832\ua838\ua83d \ud83c\udc3a\ud83c\udc9f\ud83c\udc4e" +
- " \ud7fb\ud7ce\ud7c6\ud7f8 erkzdwed ftzpcx ecbylf geptanc jxbhc ophh wqng \ue48c\u9c86Z imkj" +
- "nwetjbz njivpvo \u6d9a\ud8da\udcba \u29f4\u29fd\u29a6\u2980\u2989\u29f3\u29ec\u2991\u29e5" +
- "\u29c6 \udb08\ude9d\u2ecb\u037e chmap <!--<sCrip \ud83c\udc34\ud83c\udc79 SoOq l botgy \ud83c" +
- "\udc11\ud83c\udc2e\ud83c\udc10 -)h?|] \ud801\udc2e\ud801\udc47 pjlrcjij lpdft v.o| qolrd b " +
- "uefppzjyvva D\u05de\u0425\u01a3\ue1c0f\uf117\ue266\u0268\u03ec ynik \udae4\udc38\udba0\udd4c" +
- " M\ue775\u000f \u3170\u3171\u3185\u3140\u3165\u317f \u07f6\u4250vp\u001c\u07a9\uba71 myfawjt" +
- "wvc cqsc o uxabwm \ua9b0\ua9d3 \u0293P\u4dde\u034e \udbe7\udd0b\udbce\udf4d a\udb4a\ude26x" +
- "\u0bc5\u0355 xtwyyxkj uiwqzvdn \u00c4\u00f4\u00b9\u00f3\u00e3 svfczoqm fx \ua65a\ua696\ua675" +
- "\ua651\ua661\ua686\ua644 cohpzoprr \u000f\ud9d5\udcbd\ud8fa\udc16\ub733\ud8d9\udcf7\uefe9" +
- "\u02da wssqee tikod iwdo cuoi mvkvd \ud834\udcb7\ud834\udc52\ud834\udc37\ud834\udc30 yqmvakwq" +
- " bwgy ndp \u0b53\u0b75\u0b60\u0b25\u0b1d\u0b1b\u0b19\u0b62 <pmg cqponju tiwlftbdn jdmc <?" +
- "</p waesof \u3fea\u43bd\u3524\u3d5b \uf87f\u03ab\\\u0727?\uf145 vwznkqkz \ud83c\ude6c\ud83c" +
- "\udea7\ud83c\udedd powof \u94f3\u0392\ue4b5$ \u079f\u07b5\u0781\u07ba\u0786\u07ae\u0782\u0785" +
- " \ud83c\udecc\ud83c\ude8e\ud83c\udea1\ud83c\ude74 \u2afb\u2a2c\u2ae8\u2ab5\u2af4 x\u4c6f hlb" +
- " oirm \ud83c\udc0f\ud83c\udc19 abzbdmkzc qsvvzzc \uf14b \udb53\ude89\u04d2\u53fe\ueb79 uuexkn" +
- " nyeevfme \ue210\uea3e zdwk licyc { cik o].o{1,5 \ua9d1\ua984\ua997\ua99d\ua9a2\ua9b3\ua986" +
- "\ua9d7 \u13ea\u13fb\u13b8\u13b9\u13db\u13e2\u13cf\u13c3\u13c8\u13cc\u13bc \ueeae\u3c1c\uf043" +
- "\u3302 \ufb791\u0009\uc0b7\u039cWG\u4f35\u000f\uf28c \ueb24\udb18\uddef\ufb2c n-jr wsmhtbq " +
- "\ue76b\ud894\udec7\u37f8 box \u1726\u1720\u172b\u173c\u1727 gqcrnrhxcj \u09f8 rof \ua7fa" +
- "\ua7a1 \u07ef\u07f3\u07e2\u07e0\u07d7 udhuv gadgqab({1 \u2c52\u2c30\u2c17\u2c16 P\u33f9\u06da" +
- "\u284b\u0734\u0799 \u289a\u28a1\u28f0\u2853\u282a\u284b\u2859\u2881\u283c qmkopt qbvxnls \ud9c6" +
- "\udc11Z\u7c39\u24ec\u0353 \u069c dfdgt \ue967\u0493\uf460 )yv-|. nl qvmu x{1,5} \\'\\' ' \u0969" +
- "\u0926\u0943\u0966\u0931\u0970\u094d\u0911\u0970 phiyfw ;\\'\\ zngsxit \u07ec&\ud914\udd55\u9ab7" +
- " ?[| b \ufffc\uffff\ufffb\ufff3\ufff7\ufff8\ufff8\ufffb\ufff5\ufff9\ufffd \u2ffd\u2ff2\u2ff1" +
- "\u2ff9\u2ff6\u2ff1\u2ff8\u2ff1\u2ff8 \ua73d\ua793\ua7d1\ua7cf \u258d\u2599\u259e\u258e\u258d" +
- "\u259f \u001fha f \u0b2e\u0b37\u0b71\u0b44\u0b40\u0b2b \uf1909\ud033 ofkl tbuxhs \ufb06\ufb47" +
- " rqcr \u043d\uf8cf\u001c \ud87e\ude05\ud87e\ude0d\ud87e\udd99\ud87e\udcc0 qqzqzntfby \u0a3f" +
- "\u0a0e\u0a16 \ud7b8\ud7cd\ud7c7\ud7cc\ud7ca\ud7e8\ud7f9\ud7b3\ud7df arzxo \u0f23\u0f2b\u0f68" +
- "\u0f1c\u0fe8\u0f97\u0f27\u0fbd 190854825 \ua915\ua907\ua902\ua902\ua907 \ufbbb\ufdd1\ufbdb" +
- "\ufbed\ufbbb\ufd81\ufd41\ufc3a rrxnh \u0ead\u0ebb\u0e97\u0eff\u0eed\u0e94\u0e86 \ud8c0\udd29" +
- "\u0016\ue050\uebf0;\u79c0\u07ba\uf8ed b \u0122\u0012\udaf5\udcfb+ mkt dluv \u18db\u18d4\u18ea" +
- " \uee53\ueb89\u0679 \u24c2\u24ee\u24e5\u24ab\u24e1\u2460 \ub41eq \uf1e0Tu\u0018\ue5b5 cqf" +
- "pwjzw cadaxx \u2642\u26be\u2603\u26aa\u26b0 pybwrfqbzr wgyejg cbhzl ipagqw \ud841\udd0d" +
- "\ud84a\udc42\ud84b\udf94\ud85e\udf91\ud847\udd41 fgjm lhtmoqls \u2cc1\u076af >\u034e\ud8a7" +
- "\udd17U\uffcf \u42cb\u07d6\u1d08Y\u0570 o\u016c] .ed]l.u oxms :\uf3cc\u0f67\u0014\u22c6" +
- "\u0720E \u1fef\u1f6f\u1f6a <scri \u63fb\u0508d\ueb2a\u001d\ue3f5\ue915\ud33d \ud800\udf43" +
- "\ud800\udf43\ud800\udf4c\ud800\udf46 \ud802\udc3c\ud802\udc00 ktdajettd urkyb \u040e\uaacf" +
- "\ufd7f\uf130\u048f\u80a6g\u0277\u0013\u8981\uc35d xckws icodgomr \udbf2\ude88\u9e5f o " +
- "h{0,5}x cu oqtvvb ohbuuew ggh 0\u001d=\u8479\ufc33\ue941\ue518 \uff87\u0012\u0226\u743d" +
- "\uef94e\ue0e2\u05cc \ue261\u0015\uf9dc\u8233\u0305/\u111e3\udbb7\udcb5 mxgbvl \uf20f\ud860" +
- "\udc00\uf9f2\uecd2 fl \u03d1\u1664\u5e1d\u619b\uda19\udfe0v/ \ud863\udfa2U\ue0c1\u07f1" +
- "\ue071\udb8f\udeb6 miz \u0641\udb66\udce0' >\ud9c0\udfaf\u07b3J\uf240\ud863\udff8\u01bf" +
- "\u2257\u008b\u0186\u0006 \uaa90\uaa92\uaa9a\uaad6\uaaa7\uaac1\uaa9d\uaaa0\uaaab vorp \u1380" +
- "\u1392\u139e\u138b\u1390\u1386 \uf626\uda58\uddb3\u0014 qrzl fzrb rllb \uc5e5\uf606\u0749" +
- "\ufff8\ud88a\udec12\ud97e\udee4 zscmiqtr \u01be\n \u05f2\u05a0\u05ca\u05de\u059d\u05ac " +
- "\u2e21\u2e62\u2e72 \u0293 \ufff0\ufff3\ufff8\uffff\ufff2 grtuzvl \ua8bc\ua880\ua89a kprmh " +
- "\ud802\ude51\ud802\ude2e\ud802\ude09\ud802\ude15 cwauqnjs Ou \u31c9\u31dc\u31e4\u31d1\u31e5" +
- "\u31c1\u31d1\u31ce\u31c8 \u31f6\u31fd\u31f0\u31fa\u31f0\u31f2\u31f3\u31f9 wpyi awanm " +
- "irnupwe 񾍅 vzwzc qhupnxm qbaboo gtxfdycm vnvaptcc \u0356\ud93f\udf7a {0,5})[k oxnum" +
- "pqyhkg \ufc2c\u0213\ue36e\u0017\ud98b\udc43 \u27f3\u27f7\u27ff\u27ff\u27f5\u27ff\u27f1 hm" +
- "kmwm j{1,5} \u0293\u0262\u2c6d\u0278\u0267\u2c64\u02a8\u0296\u0284 thjrit \u12e3 \ud808" +
- "\udf7d\ud808\udca7 b prkjpdeavdooly \"\\\u06d5\ud9dc\uddb6;\ufdd6\u05bd\u077f kyql \u2d2e" +
- "\u2d04\u2d2e\u2d2a\u2d03\u2d1d scjl higfadu \u3041\u306c\u3073\u305c\u308a\u308e\u3075" +
- "\u3086 akfmdqrrwkw rfmqcxyekf \ud86c\udd70\ud86c\udcdc\ud86b\udea2 c< cwab t \ud800\udd13" +
- "\ud800\udd23 \u0138\ud817\uddcd\uf9f2 zisakv \uea3e\u0314\u07be\ufe67b\ud38b\u0439\r " +
- "\ua766\ua7c5\ua769\ua7a8\ua794 ksfyrg ({1,5}j)?wl \ua94a\ua943\ua932\ua939\ua946\ua95c" +
- "\ua955\ua952\ua958\ua94c pshdyg lhxpypaug blqtrdsvc wycmf ndrzh ekvdkm bnnceq napak n Ko" +
- "KomfgoU \ud83c\uded0\ud83c\udeee \n-->169 mopdhsp \uda82\udca1\\T\udb22\udea8\ufa82C\"" +
- "\u06d9\u0218 \u8708 \u18cd\u18c0\u18e8\u18fc\u18be\u18fd\u18c0 yldjbofabwj \u1720\u1739" +
- "\u1729 ([[m{1,5} blqaoru pvsvfall ydsz \ufd6f\ufce2\ufd4d\ufd07\ufde5\ufddc\ufb6c\ufbc9" +
- "\ufd14\ufc4f\ufd05 \u216b\u218a\u2152\u2172\u217d\u2181\u2188 savpwhs {1,5}f[ha-y[) xnzz " +
- "gksck \u783a\u517a\u513e\u7355\u8741 kicgsn \u3117\u311c\u3104\u310c\u312e\u3104\u3103 " +
- "\u0291\u430b\uc9bfd\ue6e1\uf2d6~0 \ud802\udd38 \ub2cd\uca67\u1c0d\u034c\uf3e2 \u03a2\u0009" +
- "\uda96\udfde \u0010\ufb41\u06dd\u06d0\ue4ef\u241b \ue1a3d\ub55d=\ud8fd\udd54\ueb5f\ud844" +
- "\udf25 xnygolayn txnlsggei yhn \u0e5c\u0e02 \\ fornos oe epp ";
-
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
- }
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
- }
- };
+ public void testCuriousMassiveString() throws Exception {
+ String text =
+ "yqt \u0728\u0707\u0712\u0720\u0734 \u204c\u201d hyipy \u2667\u2619"
+ + "\u26ec\u267b\u26da uboyjwfbv \u2012\u205d\u2042\u200a\u2047\u2040 gyxmmz yvv %"
+ + "\ufb86 \n<script> hupvobbv jvsztd x rww ct{1,5} brteyi dfgyzqbm hdykd ahgeizyhv"
+ + " kLn c#\ud8f8\udd74 fPezd ktedq \ufcea=\ud997\uddc9\u876bJ\u0231\ud98a\udce0\uf872"
+ + " zquqah \ub9d6\u144e\uc686 3\ud93d\udfca\u1215\ud614 tnorask \u0348\u0334\u0334"
+ + "\u0300\u033d geqdeoghh foojebut \ufb52\u227ag\ud9bd\udc3a\u07efK nyantr lksxw fs"
+ + "zies ubzqfolksjpgk \u6fa3\ud859\udc36\u0501\ucca0\u0306\u001e\ua756\u052f \ucaf7"
+ + "\u0247\u0009\ufdddg \ud83c\udd02\ud83c\uddaf \u05628\u2b49\u02e3\u0718\u0769\u4f1b"
+ + "\u0496\u0766\uecaa\ufb44 \u001d \u0006hr\u00f0\ue649\u041a\uda6f\udfa5\uf31b\ue274"
+ + " ptgjf \ud8cc\udf83M\u0013\u04c6i \u205f\u2004\u2032\u2001\u2057\u2066 \u07d0\uacdb"
+ + "\u06a5z pqfxwgbwe \ud1bc\u2eba\u2d45\u02ee\u56df xnujtfs \u1b19\u1b17\u1b39\u1b20"
+ + "\u1b69\u1b58\u1b03\u1b6e\u1b73\u1b20 afsl zxlzziqh ahrhckhktf \ud801\udc5b\ud801\udc61"
+ + " bkpmeyyqobwi qnkunmpjpihezll plhhws \u37f7\u41d6\u3dca\u3e80\u4923\u36b8 \u195a\u1959"
+ + "\u196f\u1968\u1958\u197f </p \u0006s\u019f\uda82\udc90H_\u079d\ufd6f: idpp \u217c"
+ + "\u2168\u2185\u2179\u2156\u2159\u217c\u2154\u2158 ({1,5}( jkieylqzmb bfirnaj \uea71"
+ + "\uf17f\u0749\u054c \ud9ea\udf83\udbea\ude91j x\u3366\u09c2\ud828\udd13~\u6fda\ueeee"
+ + " \ud834\udcd3\ud834\udc2b\ud834\udc8b\ud834\udcd8 dl \ud802\ude3a\ud802\ude36\ud802"
+ + "\ude23\ud802\ude56 \u20ad\u20a0\u20a0\u20a0\u20b8\u20b4\u20ad lcql \u0547\u0156]"
+ + "\ue344V] \ud90c\udd7d\u000eS\uf965\uf15e\u008f qdn \udac3\ude3c buy m qni \u31a4\u31a3"
+ + "\u31b2\u31a9\u31a7 \u20df\u20ee\u20d3\u20d4\u20f1\u20de \u4dc1\u4dc4\u4dde\u4dec\u4df9"
+ + "\u4dee\u4dc5 \udb40\udc36 gstfqnfWY \u24ab\u8d69\u0301 n?]v||]i )- \udb0b\ude77\uf634"
+ + "\u0762 tpkkjlbcntsk eebtzirw xo hktxy \n</ vxro xpr mtlp p|tjf|i?.- lxpfo \udbd7"
+ + "\udf78\udbf5\udf57 u..b jjj]p?e jtckvhqb \u20eb\u20ea\u20fa\u20ef\u20e1\u20ed\u20eb vvm"
+ + "uhbsvyi jx mqwxckf \u00d7 qqzs \u05ae\u022d\udb7c\udfb1\u070b vnhkz egnutyuat \udaa2"
+ + "\udf20\ufa45#O\u2b61\u0d0e \u09a2\u0996\u09cd\u09b4 v \udbdb\ude9bqp owsln \ua837\ua833"
+ + "\ua83f\ua83f\ua83f\ua831\ua83c\ua839 \u1a15\u1a1c\u1a12 \ud83c\ude20 >&pt&#x>129 \ud9f1"
+ + "\udf8c\uecdd \ud809\udc48\ud809\udc72 wswskop \ua70d\ua70e ohcjsboriux \u2025\u2000\u206e"
+ + "\u2039\u2036\u2002\u200e \u1741\u1741\u175e bsykot \u0696\ufab5W ifj cnosdrAxa qtv wvrdn"
+ + " dmt (){0 udqg Z5U\ufcab\ue9ed\u0378 ts zndsksqtu uxbkAbn \u0535\"\u001b\ud923\udcc3\udae7"
+ + "\udccf\u02ac \u0131\ua72e\u1273\u02f8\u2329\u5d83Q icakme oojxv hpec xndjzxre )e{0,5}. "
+ + "\u0005!\u0009\u0004\u000bt\u0006N vqxnokp cdshqof \ua654\ua681\ua667\ua676\ua698 vwwp "
+ + "\ufe4b\ufe3b\ufe4d\ufe42\ufe44\ufe38\ufe4b\ufe4c\ufe4b\ufe38 \u2ffb\u2ffa\u2ff6\u2ff7"
+ + "\u2ffc\u2ff6\u2ff5 \u3116\u312c\u3100\u3102\u310f\u3116 agyueiij \u0764\u1178\ud866\udca1"
+ + "\u00f2\u039d \ud802\udc12\ud802\udc1e\ud802\udc28\ud802\udc19 kygiv fxctjyj \n"
+ + " omu \ud855\udeb1\u063c\ufd54\u9dbf\uf80a\ufc60 \u76ee\u3365\uf998\u70a8\u02d2 \u05e5"
+ + "\u0478\u61bb \ua71c\ua708\ua70d\ua71a\ua712\ua712\ua719\ua706\ua71f\ua71c\ua71e\ua718 pgq"
+ + "arvtzcoduk pyok \u1779\u1766\u1778\u177b \u16e5\u16a7\u16f3\u16fe\u16c8\u16ba\u16a4 \u0527"
+ + "\u052f\u0513\u0500 iisgh hxd \u13dd\u13c6\u13db\u13ee\u13d7 \u0019\ufed4 \ud802\udd3c\ud802"
+ + "\udd30\ud802\udd3d\ud802\udd24\ud802\udd2c jm gvylt eax xsbt mvuvnw \u0246\u5514\udb16\uddcf"
+ + "\u1dc2\ud07b\u07af\u12e8\u8e8f\u0004 phy haduzzw \u04a1\u8334\u14b5\uf0de\udb4b\udec0\u6b69 "
+ + "dubdl \u0cd2\u06c7\uf0297\u45efy-\u05e9\u01a3\uf46f aiafsh &# \u0003\ue5ab\uedcd] xhz vil"
+ + "wdlytsj \uda10\udf4f\u87b2 tomuca \u1fe4\u1f4c\u1fab \u035d\u0332 xgeel nzp -)r)]r([{ nbc "
+ + "\u01b4\ud80f\udff5\u0008\u0091 tbugdgv \ud9cc\udd57\u0776t\uda0f\udc17 rsuwxqxm \u2d71\u2d3d"
+ + "\u2d6e zsvuicin \udb50\ude9d\u7424\u30c7\uff73\ue11e\u0014 qxtxx dlssfvxg ud \u0c1f\ud9d9"
+ + "\udce4\u2317\u0482\u017b \ud83c\udd91\ud83c\uddaf\ud83c\udd84 \ud834\udf7d\ud834\udf70\ud834"
+ + "\udf61 \uabfc\uabe1\uabcd\uabd1\uabe8\uabf9 \u292a\u2916\u295d\u290b\u296d\u291f \uf476\u0283"
+ + "\u03d5\ucfe2 h)(o? lqeatt \u20c9\u20a5\u20cd \u1634d\u001a\ua052:\u00db \ud83c\udc8a\ud83c"
+ + "\udc41\ud83c\udc8a\ud83c\udc6e k civyjm ptz \uf20d\uea32&\ud8db\ude48\uf657s\u06dc\u9aa5\udbd7"
+ + "\udc74\u0111 cehuo \u2090\u209b\u2099\u208c\u209a\u2088\u2092\u207e\u207b\u2089 efpifnvdd zstz"
+ + "duuyb \u04af3 \u2e5f\u2e7e\u2e7c\u2e54\u2e0f\u2e68\u2e0d\u2e05 <??> \u28d3\udbb7\udf6fJ\uf089"
+ + "\ub617\ufb80\u07d0\uf141 \u0820\u083b\u0800\u0801\u0832\u0823\u0820\u081c\u0835 r laxzpfbcvz "
+ + "iuwbmq scpeqaq nvw{1,5}s) \u234c\u231f\u231e\u23cc\u23d8\u2302\u2345\u231b\u239d\u231e 154614 "
+ + "wgefnmgq \udbbe\udc2a\uee8c ayaeg \u243c\u2404\u2409\u241a dd hxuawoswx jqghoznw \u0019\u70cd& "
+ + "\ufe0a\ufe02\ufe04\ufe0c\ufe0d \u0510\u0523\u0510\u0501\u0508\u0513\u050e ropgywv guqsrcz kmf "
+ + "\u0d1f\u0d5c\u0d24\u0d5f\u0d0b\u0d14 xgklnql oprw \u0365\u034e\u036a\u0310\u034f\u0316\u031f "
+ + "\u01b2\u55f6\uf1eeo\ue994\u07c4? wewz idjwsqwo \ufe67\ufe66\ufe52\ufe6a\ufe5b\ufe57\ufe68\ufe62"
+ + " \u27fd\u27fe\u27f6\u27f8 fcsqqvoy edrso \u5580\ue897 vantkitto sm \uff60\ud957\udf48\uf919w"
+ + "\ud889\udf3e\ud9c8\uddf6 jhas uqzmlr \ua4ce\ua4aa\ua4b3\ua4b5\ua4c2\ua4a5 kvuop ><script> "
+ + "\ud802\udd0f\ud802\udd15\ud802\udd11 \u16bc gjyabb mlaewr \u1175\u11cc\u1192 ri ]\u01eb\ue4ca"
+ + " \uedca\ucd66\u597c\u03df\uaf8f\u0619 &#x; ]?e \ua6c2\ua6ed\ua6eb\ua6ea\ua6cd\ua6e2 gfpafsr"
+ + " pooo \u20cc\u20c4\u20a7\u20c8\u20a6\u20b0\u20ad \udb40\udd5b tgcdmeuau \u141f\u637d \ufba8"
+ + "\ufcc7\ufde1\ufc85\ufbfc\ufbed b \u319a\u3193\u3192\u3195\u319e\u319d\u3195\u3194\u3197\u319c "
+ + "\u0514\u0503\u051c\u050e\u0502\u0520 \u1db3\u1db5\u1d96 \ud977\udde8w\u02ec&\u01dd\u29ed\ufead"
+ + "y\u03e3 ukzqq {0,5}{0 \u000f\uf028$\u046f7\udb7e\uded2 <p><l \uea5e(\uf1dcX\u376b ([mi[n(a Jev"
+ + "xsJl \ubd09\u04c1\ua0f3\uff7b \u1cd3\u1cd9\u1cf6\u1cf0\u1cd8\u1cdd\u1cdd\u1ce7\u1ce3\u1cd9 "
+ + "\ud800\udf91\ud800\udf99\ud800\udf83 \"135541 \u18b3\u18c0\u18c2\u18ea\u18c4\u18fe\u18b2\u18fd"
+ + "\u18c3 uwykvfd lqiflow afdfctcz ol[hemp strmhxmk \ua732\ua748\ua731\ua791\ua78b\ua7ee\ua7ea"
+ + "\ua789\ua766\ua7e4 gmzpmrpzr dqfp wfxwjzepdj M\udb03\udeff\u13c5 afsncdwp \ue716\u0734\ud8f9"
+ + "\udc986\u0017\u0211 t r vhczf (]|q.{0 \u195e\u1973\u1958\u1956\u196c\u1973\u1966\u196c\u197a"
+ + " \u2595\u258e\u259a\u2591\u2583\u2595 l kgopyql wes E\u6611\ub713\u7058^ bipq dx 7507 \ua90b"
+ + "\ua90b ktjeqqx \u0e1d\u0e7f\u0e35 #(\u71b7'\u06e5\u03e4(\uf714\u6ff2 advbgh \u319c\u3191 \uef11"
+ + "% \uc8a7C\ud8ed\udf4c rjb \u02ca\uf5bd\ue379n \ud834\udd7d\ud834\udd83 jracjcd rpearfd ujzvdc"
+ + " ofg \u09df\u09f4\u0980\u09b3\u09bf\u09b7 \ud9cc\uddf4$\udb08\udf72 iqcnwsyjmo </scri ]-q jsit"
+ + "gjg naeajiix vvmq dnlihau o cgyp tqsfe uixlzmhz sixhftpr uvtbvv mphcWojZs \u190b\ud9c3\ude7c"
+ + "\u008b\u0606\ua8b1 a \u000ebq# \u1e57\u0b66\uda41\ude32\ubfd6 ohph b- ze \ue2a9\u0000 zatuye"
+ + " \ufd26\ufdfa\ufbbf\ufdb4\ufde3\ufd14\ufc25\ufcb8 sbtpb nxtacgjo \ud834\ude2a\ud834\ude0f"
+ + "\ud834\ude14\ud834\ude27 \ua835\ua835 ujpjkkyhujx \u001e\ud9a7\udc45\u0011>\u1aef\u0d0d <"
+ + " hcefg \u01f0\u01d3 gxlwv \ud802\udd2f\ud802\udd34 \udb9c\udcc8\udbb6\ude1e\udbaf\ude33\udbae"
+ + "\udc49 xyzlzynd \ud83c\udd44 vynhdwh \u315d\u3157\u314d\u3180\u317d\u318d\u317d\u3156 ?>\" "
+ + "\ud800\udfdb\ud800\udfb8\ud800\udfa6\ud800\udfa7 hszn sspwldpdwjfhk vrbag \ueabd\ud9f2\udfb5"
+ + "\udafb\udd28\uf6a4y\ufdeb \u0798\u078f\u0789 \ue80c\uf5c1\u001a\ud84b\uddef ywdrudo odqsts nnp"
+ + "vfl nvotk rdwhr rquj cn \ud7d4\ud7b3\ud7c7\ud7bf\ud7bd � emdq pnqsbbgzs \u2c7d\u2c7e fj"
+ + "kwhku >&c \ud800\udf85\ud800\udf88\ud800\udf93\ud800\udf84\ud800\udf82\ud800\udf8b '\n\"<p></p>"
+ + " xltgxaa vqjmhiz n .m]c]tr( qerokel lc kugmimmtfxi <?� [g)|n|[ cij </ BwbZfg "
+ + "pta bmhn \uf60dz\u54ca fwdp gnkz \u030ba\ue115}\udb7d\ude86\ud813\udc50\uedb9 \u1745\u1749\u174d"
+ + "\u1747\u174b\u174f \ud802\udf09\ud802\udf3f\ud802\udf03\ud802\udf00 agmj \u1db7 \u60669\u000f"
+ + "j\u000f\u02e4e\u05f5f p \udaac\udc9f\uc257\u6e58U zcwsy \u19a7\u19cf\u19aa\u199f\u19b7 zhll"
+ + " sbmv .uud \u040c\u2e78\ud9fc\udd0d\ufb7f\uf2e1\u04bf vqkyxua \ud834\udf5e\ud834\udf45\ud834"
+ + "\udf23 \ud834\uddbe\ud834\udd9b\ud834\uddc4 f [{0,5}t ovbsy tcj nyalqyabn yzhes mlwpw \ud835"
+ + "\uddd5\ud835\udfdf\ud835\uddb4\ud835\ude3e epqfkk cssusatbbq \u1424\u1413\u1652\u15f0 dtfy zN"
+ + " \u2417\u2435\u2407 qtz \u2fff\u2ff1\u2ff8\u2ff8\u2ff7\u2ff7 \ud802\ude43 gfpe stimmb ugPwlr "
+ + "\u0737\u0712\u0712\u071c \u21de \u01d8\u079e\u8215\ue5b9\u07ef\u3cff\u0478 \u05dd\u05e5 gwqset"
+ + "iyyh jbncv \u68ba\u52aa) kcoeiuogjsh n[nh- \uf5ff\u7ec3Z zrptjltqstzvx ts tbod 𕠗 \u07fd"
+ + "\u07c1\u07c0\u07e9\u07fa\u07f2\u07e3\u07e8\u07cb\u07eb\u07d8 fisslh \ue40f\u012b\u02cf\u0766"
+ + " \u1c25\u1c4f\u1c1d\u1c20 \"--> dPhwz \ud808\udef9\ud808\udf4a\ud808\uddd3 cicqhk D\ue7d3=\u5228"
+ + "\udbc3\udd18\ueb0d\u0012\u0744\ufb04U\u001e\uf388c\u0306 \u2c08\u2c1e Xxzcd \u001d\u0230\u45e8"
+ + "\u0653 <zlx \u1e8f\u1e28\u1e3c\u1e8d\u1ee8\u1e69 zcoeork d gusesc \ud802\udd3e nyr vkfrgi \u2cde"
+ + " mo nklnsqweh <script gtoppwsbik vrcdbsx pz \udb0d\ude0c|\u93d0\uf241\u28a8\u0531 \ud83c"
+ + "\udc2b\ud83c\udc10 \ud800\udf91\ud800\udf8e qwmmkty \u19f7\u19f7\u19e8\u19e0\u19f9\u19f6\u19e6"
+ + " \u7a60\u7b7b\u878c\u603c\u53c6\u6552\u6dfe \u0776\u0775 foxaqi m cdjd guyqxodsccy \ucd7d\ub5eb"
+ + "\ud29e\ub9ad\uba00\uac9d\ud2f0 pxjtj \ue362\u079fx\uf193 ){1,5}[{ hmpujw \u3193\u319b\u3195"
+ + "\u319c\u3198\u3193\u3195\u319d\u319e \udb40\udd65\udb40\udd29\udb40\udd5c\udb40\uddba \ud801"
+ + "\udc18\ud801\udc24\ud801\udc4f\ud801\udc15\ud801\udc04 \u1970\u196c\u1963\u196f\u1979 vjpeg "
+ + "\ufeb9 lhoeh 吋 Szu \u0c21\u0c21\u0c36\u0c0e oyb \u1c7c\ue0ba\u001e gskf \ud826\udd47"
+ + "\u0018 ooxki \u001d\u5b0d \uf0e2\u05ba\u000e\u6200 \u2ecc\u2e8a\u2eb8\u2ee5\u2edb\u2ee4\u2ec7"
+ + "\u2ef9\u2e9e\u2e99 xpyijb bqvtjfjrxkjv sruyitq jobm u \u0752\u075d\u0778\u0754\u075c \ua4af"
+ + "\ua4a5\ua4be\ua4a6\ua4b9\ua4b9 \ua835\ua832\ua838\ua83d \ud83c\udc3a\ud83c\udc9f\ud83c\udc4e"
+ + " \ud7fb\ud7ce\ud7c6\ud7f8 erkzdwed ftzpcx ecbylf geptanc jxbhc ophh wqng \ue48c\u9c86Z imkj"
+ + "nwetjbz njivpvo \u6d9a\ud8da\udcba \u29f4\u29fd\u29a6\u2980\u2989\u29f3\u29ec\u2991\u29e5"
+ + "\u29c6 \udb08\ude9d\u2ecb\u037e chmap <!--<sCrip \ud83c\udc34\ud83c\udc79 SoOq l botgy \ud83c"
+ + "\udc11\ud83c\udc2e\ud83c\udc10 -)h?|] \ud801\udc2e\ud801\udc47 pjlrcjij lpdft v.o| qolrd b "
+ + "uefppzjyvva D\u05de\u0425\u01a3\ue1c0f\uf117\ue266\u0268\u03ec ynik \udae4\udc38\udba0\udd4c"
+ + " M\ue775\u000f \u3170\u3171\u3185\u3140\u3165\u317f \u07f6\u4250vp\u001c\u07a9\uba71 myfawjt"
+ + "wvc cqsc o uxabwm \ua9b0\ua9d3 \u0293P\u4dde\u034e \udbe7\udd0b\udbce\udf4d a\udb4a\ude26x"
+ + "\u0bc5\u0355 xtwyyxkj uiwqzvdn \u00c4\u00f4\u00b9\u00f3\u00e3 svfczoqm fx \ua65a\ua696\ua675"
+ + "\ua651\ua661\ua686\ua644 cohpzoprr \u000f\ud9d5\udcbd\ud8fa\udc16\ub733\ud8d9\udcf7\uefe9"
+ + "\u02da wssqee tikod iwdo cuoi mvkvd \ud834\udcb7\ud834\udc52\ud834\udc37\ud834\udc30 yqmvakwq"
+ + " bwgy ndp \u0b53\u0b75\u0b60\u0b25\u0b1d\u0b1b\u0b19\u0b62 <pmg cqponju tiwlftbdn jdmc <?"
+ + "</p waesof \u3fea\u43bd\u3524\u3d5b \uf87f\u03ab\\\u0727?\uf145 vwznkqkz \ud83c\ude6c\ud83c"
+ + "\udea7\ud83c\udedd powof \u94f3\u0392\ue4b5$ \u079f\u07b5\u0781\u07ba\u0786\u07ae\u0782\u0785"
+ + " \ud83c\udecc\ud83c\ude8e\ud83c\udea1\ud83c\ude74 \u2afb\u2a2c\u2ae8\u2ab5\u2af4 x\u4c6f hlb"
+ + " oirm \ud83c\udc0f\ud83c\udc19 abzbdmkzc qsvvzzc \uf14b \udb53\ude89\u04d2\u53fe\ueb79 uuexkn"
+ + " nyeevfme \ue210\uea3e zdwk licyc { cik o].o{1,5 \ua9d1\ua984\ua997\ua99d\ua9a2\ua9b3\ua986"
+ + "\ua9d7 \u13ea\u13fb\u13b8\u13b9\u13db\u13e2\u13cf\u13c3\u13c8\u13cc\u13bc \ueeae\u3c1c\uf043"
+ + "\u3302 \ufb791\u0009\uc0b7\u039cWG\u4f35\u000f\uf28c \ueb24\udb18\uddef\ufb2c n-jr wsmhtbq "
+ + "\ue76b\ud894\udec7\u37f8 box \u1726\u1720\u172b\u173c\u1727 gqcrnrhxcj \u09f8 rof \ua7fa"
+ + "\ua7a1 \u07ef\u07f3\u07e2\u07e0\u07d7 udhuv gadgqab({1 \u2c52\u2c30\u2c17\u2c16 P\u33f9\u06da"
+ + "\u284b\u0734\u0799 \u289a\u28a1\u28f0\u2853\u282a\u284b\u2859\u2881\u283c qmkopt qbvxnls \ud9c6"
+ + "\udc11Z\u7c39\u24ec\u0353 \u069c dfdgt \ue967\u0493\uf460 )yv-|. nl qvmu x{1,5} \\'\\' ' \u0969"
+ + "\u0926\u0943\u0966\u0931\u0970\u094d\u0911\u0970 phiyfw ;\\'\\ zngsxit \u07ec&\ud914\udd55\u9ab7"
+ + " ?[| b \ufffc\uffff\ufffb\ufff3\ufff7\ufff8\ufff8\ufffb\ufff5\ufff9\ufffd \u2ffd\u2ff2\u2ff1"
+ + "\u2ff9\u2ff6\u2ff1\u2ff8\u2ff1\u2ff8 \ua73d\ua793\ua7d1\ua7cf \u258d\u2599\u259e\u258e\u258d"
+ + "\u259f \u001fha f \u0b2e\u0b37\u0b71\u0b44\u0b40\u0b2b \uf1909\ud033 ofkl tbuxhs \ufb06\ufb47"
+ + " rqcr \u043d\uf8cf\u001c \ud87e\ude05\ud87e\ude0d\ud87e\udd99\ud87e\udcc0 qqzqzntfby \u0a3f"
+ + "\u0a0e\u0a16 \ud7b8\ud7cd\ud7c7\ud7cc\ud7ca\ud7e8\ud7f9\ud7b3\ud7df arzxo \u0f23\u0f2b\u0f68"
+ + "\u0f1c\u0fe8\u0f97\u0f27\u0fbd 190854825 \ua915\ua907\ua902\ua902\ua907 \ufbbb\ufdd1\ufbdb"
+ + "\ufbed\ufbbb\ufd81\ufd41\ufc3a rrxnh \u0ead\u0ebb\u0e97\u0eff\u0eed\u0e94\u0e86 \ud8c0\udd29"
+ + "\u0016\ue050\uebf0;\u79c0\u07ba\uf8ed b \u0122\u0012\udaf5\udcfb+ mkt dluv \u18db\u18d4\u18ea"
+ + " \uee53\ueb89\u0679 \u24c2\u24ee\u24e5\u24ab\u24e1\u2460 \ub41eq \uf1e0Tu\u0018\ue5b5 cqf"
+ + "pwjzw cadaxx \u2642\u26be\u2603\u26aa\u26b0 pybwrfqbzr wgyejg cbhzl ipagqw \ud841\udd0d"
+ + "\ud84a\udc42\ud84b\udf94\ud85e\udf91\ud847\udd41 fgjm lhtmoqls \u2cc1\u076af >\u034e\ud8a7"
+ + "\udd17U\uffcf \u42cb\u07d6\u1d08Y\u0570 o\u016c] .ed]l.u oxms :\uf3cc\u0f67\u0014\u22c6"
+ + "\u0720E \u1fef\u1f6f\u1f6a <scri \u63fb\u0508d\ueb2a\u001d\ue3f5\ue915\ud33d \ud800\udf43"
+ + "\ud800\udf43\ud800\udf4c\ud800\udf46 \ud802\udc3c\ud802\udc00 ktdajettd urkyb \u040e\uaacf"
+ + "\ufd7f\uf130\u048f\u80a6g\u0277\u0013\u8981\uc35d xckws icodgomr \udbf2\ude88\u9e5f o "
+ + "h{0,5}x cu oqtvvb ohbuuew ggh 0\u001d=\u8479\ufc33\ue941\ue518 \uff87\u0012\u0226\u743d"
+ + "\uef94e\ue0e2\u05cc \ue261\u0015\uf9dc\u8233\u0305/\u111e3\udbb7\udcb5 mxgbvl \uf20f\ud860"
+ + "\udc00\uf9f2\uecd2 fl \u03d1\u1664\u5e1d\u619b\uda19\udfe0v/ \ud863\udfa2U\ue0c1\u07f1"
+ + "\ue071\udb8f\udeb6 miz \u0641\udb66\udce0' >\ud9c0\udfaf\u07b3J\uf240\ud863\udff8\u01bf"
+ + "\u2257\u008b\u0186\u0006 \uaa90\uaa92\uaa9a\uaad6\uaaa7\uaac1\uaa9d\uaaa0\uaaab vorp \u1380"
+ + "\u1392\u139e\u138b\u1390\u1386 \uf626\uda58\uddb3\u0014 qrzl fzrb rllb \uc5e5\uf606\u0749"
+ + "\ufff8\ud88a\udec12\ud97e\udee4 zscmiqtr \u01be\n \u05f2\u05a0\u05ca\u05de\u059d\u05ac "
+ + "\u2e21\u2e62\u2e72 \u0293 \ufff0\ufff3\ufff8\uffff\ufff2 grtuzvl \ua8bc\ua880\ua89a kprmh "
+ + "\ud802\ude51\ud802\ude2e\ud802\ude09\ud802\ude15 cwauqnjs Ou \u31c9\u31dc\u31e4\u31d1\u31e5"
+ + "\u31c1\u31d1\u31ce\u31c8 \u31f6\u31fd\u31f0\u31fa\u31f0\u31f2\u31f3\u31f9 wpyi awanm "
+ + "irnupwe 񾍅 vzwzc qhupnxm qbaboo gtxfdycm vnvaptcc \u0356\ud93f\udf7a {0,5})[k oxnum"
+ + "pqyhkg \ufc2c\u0213\ue36e\u0017\ud98b\udc43 \u27f3\u27f7\u27ff\u27ff\u27f5\u27ff\u27f1 hm"
+ + "kmwm j{1,5} \u0293\u0262\u2c6d\u0278\u0267\u2c64\u02a8\u0296\u0284 thjrit \u12e3 \ud808"
+ + "\udf7d\ud808\udca7 b prkjpdeavdooly \"\\\u06d5\ud9dc\uddb6;\ufdd6\u05bd\u077f kyql \u2d2e"
+ + "\u2d04\u2d2e\u2d2a\u2d03\u2d1d scjl higfadu \u3041\u306c\u3073\u305c\u308a\u308e\u3075"
+ + "\u3086 akfmdqrrwkw rfmqcxyekf \ud86c\udd70\ud86c\udcdc\ud86b\udea2 c< cwab t \ud800\udd13"
+ + "\ud800\udd23 \u0138\ud817\uddcd\uf9f2 zisakv \uea3e\u0314\u07be\ufe67b\ud38b\u0439\r "
+ + "\ua766\ua7c5\ua769\ua7a8\ua794 ksfyrg ({1,5}j)?wl \ua94a\ua943\ua932\ua939\ua946\ua95c"
+ + "\ua955\ua952\ua958\ua94c pshdyg lhxpypaug blqtrdsvc wycmf ndrzh ekvdkm bnnceq napak n Ko"
+ + "KomfgoU \ud83c\uded0\ud83c\udeee \n-->169 mopdhsp \uda82\udca1\\T\udb22\udea8\ufa82C\""
+ + "\u06d9\u0218 \u8708 \u18cd\u18c0\u18e8\u18fc\u18be\u18fd\u18c0 yldjbofabwj \u1720\u1739"
+ + "\u1729 ([[m{1,5} blqaoru pvsvfall ydsz \ufd6f\ufce2\ufd4d\ufd07\ufde5\ufddc\ufb6c\ufbc9"
+ + "\ufd14\ufc4f\ufd05 \u216b\u218a\u2152\u2172\u217d\u2181\u2188 savpwhs {1,5}f[ha-y[) xnzz "
+ + "gksck \u783a\u517a\u513e\u7355\u8741 kicgsn \u3117\u311c\u3104\u310c\u312e\u3104\u3103 "
+ + "\u0291\u430b\uc9bfd\ue6e1\uf2d6~0 \ud802\udd38 \ub2cd\uca67\u1c0d\u034c\uf3e2 \u03a2\u0009"
+ + "\uda96\udfde \u0010\ufb41\u06dd\u06d0\ue4ef\u241b \ue1a3d\ub55d=\ud8fd\udd54\ueb5f\ud844"
+ + "\udf25 xnygolayn txnlsggei yhn \u0e5c\u0e02 \\ fornos oe epp ";
+
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(
+ reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ }
+ };
for (int i = 0; i < 25; i++) {
checkAnalysisConsistency(random(), a, false, text);
}
@@ -425,17 +483,19 @@ public class TestICUNormalizer2CharFilter extends BaseTokenStreamTestCase {
public void testVeryLargeInputOfNonInertChars() throws Exception {
char[] text = new char[1000000];
Arrays.fill(text, 'a');
- try (Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new KeywordTokenizer());
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new ICUNormalizer2CharFilter(reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
- }
- }) {
+ try (Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new KeywordTokenizer());
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ICUNormalizer2CharFilter(
+ reader, Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
+ }
+ }) {
checkAnalysisConsistency(random(), a, false, new String(text));
}
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilterFactory.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilterFactory.java
index a1e01ba..217a6c9 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilterFactory.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilterFactory.java
@@ -16,25 +16,24 @@
*/
package org.apache.lucene.analysis.icu;
-
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
/** basic tests for {@link ICUNormalizer2CharFilterFactory} */
public class TestICUNormalizer2CharFilterFactory extends BaseTokenStreamTestCase {
-
+
/** Test nfkc_cf defaults */
public void testDefaults() throws Exception {
Reader reader = new StringReader("This is a Ｔｅｓｔ");
- ICUNormalizer2CharFilterFactory factory = new ICUNormalizer2CharFilterFactory(new HashMap<String,String>());
+ ICUNormalizer2CharFilterFactory factory =
+ new ICUNormalizer2CharFilterFactory(new HashMap<String, String>());
reader = factory.create(reader);
TokenStream stream = whitespaceMockTokenizer(reader);
- assertTokenStreamContents(stream, new String[] { "this", "is", "a", "test" });
+ assertTokenStreamContents(stream, new String[] {"this", "is", "a", "test"});
}
/** Test nfkc form */
@@ -45,18 +44,24 @@ public class TestICUNormalizer2CharFilterFactory extends BaseTokenStreamTestCase
ICUNormalizer2CharFilterFactory factory = new ICUNormalizer2CharFilterFactory(args);
reader = factory.create(reader);
TokenStream stream = whitespaceMockTokenizer(reader);
- assertTokenStreamContents(stream, new String[] { "This", "is", "a", "Test" });
+ assertTokenStreamContents(stream, new String[] {"This", "is", "a", "Test"});
}
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new ICUNormalizer2CharFilterFactory(new HashMap<String,String>() {{
- put("bogusArg", "bogusValue");
- }});
- });
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ new ICUNormalizer2CharFilterFactory(
+ new HashMap<String, String>() {
+ {
+ put("bogusArg", "bogusValue");
+ }
+ });
+ });
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
-
+
// TODO: add tests for different forms
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
index e38b997..e26db30 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
@@ -16,35 +16,31 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Normalizer2;
import java.io.IOException;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import com.ibm.icu.text.Normalizer2;
-
-/**
- * Tests the ICUNormalizer2Filter
- */
+/** Tests the ICUNormalizer2Filter */
public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
Analyzer a;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
- a = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
- }
- };
+ a =
+ new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
+ }
+ };
}
-
+
@Override
public void tearDown() throws Exception {
a.close();
@@ -53,55 +49,59 @@ public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
public void testDefaults() throws IOException {
// case folding
- assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" });
+ assertAnalyzesTo(a, "This is a test", new String[] {"this", "is", "a", "test"});
// case folding
- assertAnalyzesTo(a, "Ruß", new String[] { "russ" });
-
+ assertAnalyzesTo(a, "Ruß", new String[] {"russ"});
+
// case folding
- assertAnalyzesTo(a, "ΜΆΪΟΣ", new String[] { "μάϊοσ" });
- assertAnalyzesTo(a, "Μάϊος", new String[] { "μάϊοσ" });
+ assertAnalyzesTo(a, "ΜΆΪΟΣ", new String[] {"μάϊοσ"});
+ assertAnalyzesTo(a, "Μάϊος", new String[] {"μάϊοσ"});
// supplementary case folding
- assertAnalyzesTo(a, "𐐖", new String[] { "𐐾" });
-
+ assertAnalyzesTo(a, "𐐖", new String[] {"𐐾"});
+
// normalization
- assertAnalyzesTo(a, "ﴳﴺﰧ", new String[] { "طمطمطم" });
+ assertAnalyzesTo(a, "ﴳﴺﰧ", new String[] {"طمطمطم"});
// removal of default ignorables
- assertAnalyzesTo(a, "क्ष", new String[] { "क्ष" });
+ assertAnalyzesTo(a, "क्ष", new String[] {"क्ष"});
}
-
+
public void testAlternate() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(
- tokenizer,
- /* specify nfc with decompose to get nfd */
- Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE)));
- }
- };
-
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(
+ tokenizer,
+ new ICUNormalizer2Filter(
+ tokenizer,
+ /* specify nfc with decompose to get nfd */
+ Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.DECOMPOSE)));
+ }
+ };
+
// decompose EAcute into E + combining Acute
- assertAnalyzesTo(a, "\u00E9", new String[] { "\u0065\u0301" });
+ assertAnalyzesTo(a, "\u00E9", new String[] {"\u0065\u0301"});
a.close();
}
-
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
}
-
+
public void testEmptyTerm() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
- return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
+ }
+ };
checkOneTerm(a, "", "");
a.close();
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java
index 237afe5..ea80a5b 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java
@@ -16,25 +16,24 @@
*/
package org.apache.lucene.analysis.icu;
-
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
/** basic tests for {@link ICUNormalizer2FilterFactory} */
public class TestICUNormalizer2FilterFactory extends BaseTokenStreamTestCase {
-
+
/** Test nfkc_cf defaults */
public void testDefaults() throws Exception {
Reader reader = new StringReader("This is a Ｔｅｓｔ");
- ICUNormalizer2FilterFactory factory = new ICUNormalizer2FilterFactory(new HashMap<String,String>());
+ ICUNormalizer2FilterFactory factory =
+ new ICUNormalizer2FilterFactory(new HashMap<String, String>());
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "this", "is", "a", "test" });
+ assertTokenStreamContents(stream, new String[] {"this", "is", "a", "test"});
}
/** Test nfkc form */
@@ -45,18 +44,24 @@ public class TestICUNormalizer2FilterFactory extends BaseTokenStreamTestCase {
ICUNormalizer2FilterFactory factory = new ICUNormalizer2FilterFactory(args);
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "This", "is", "a", "Test" });
+ assertTokenStreamContents(stream, new String[] {"This", "is", "a", "Test"});
}
-
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new ICUNormalizer2FilterFactory(new HashMap<String,String>() {{
- put("bogusArg", "bogusValue");
- }});
- });
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ new ICUNormalizer2FilterFactory(
+ new HashMap<String, String>() {
+ {
+ put("bogusArg", "bogusValue");
+ }
+ });
+ });
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
-
+
// TODO: add tests for different forms
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
index 2f04d65..61f51b1 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
@@ -16,51 +16,46 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.Transliterator;
+import com.ibm.icu.text.UnicodeSet;
import java.io.IOException;
import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-
-import com.ibm.icu.text.Transliterator;
-import com.ibm.icu.text.UnicodeSet;
-
-/**
- * Test the ICUTransformFilter with some basic examples.
- */
+/** Test the ICUTransformFilter with some basic examples. */
public class TestICUTransformFilter extends BaseTokenStreamTestCase {
-
+
public void testBasicFunctionality() throws Exception {
- checkToken(Transliterator.getInstance("Traditional-Simplified"),
- "簡化字", "简化字");
- checkToken(Transliterator.getInstance("Katakana-Hiragana"),
- "ヒラガナ", "ひらがな");
- checkToken(Transliterator.getInstance("Fullwidth-Halfwidth"),
- "アルアノリウ", "ｱﾙｱﾉﾘｳ");
- checkToken(Transliterator.getInstance("Any-Latin"),
- "Αλφαβητικός Κατάλογος", "Alphabētikós Katálogos");
- checkToken(Transliterator.getInstance("NFD; [:Nonspacing Mark:] Remove"),
- "Alphabētikós Katálogos", "Alphabetikos Katalogos");
- checkToken(Transliterator.getInstance("Han-Latin"),
- "中国", "zhōng guó");
+ checkToken(Transliterator.getInstance("Traditional-Simplified"), "簡化字", "简化字");
+ checkToken(Transliterator.getInstance("Katakana-Hiragana"), "ヒラガナ", "ひらがな");
+ checkToken(Transliterator.getInstance("Fullwidth-Halfwidth"), "アルアノリウ", "ｱﾙｱﾉﾘｳ");
+ checkToken(
+ Transliterator.getInstance("Any-Latin"), "Αλφαβητικός Κατάλογος", "Alphabētikós Katálogos");
+ checkToken(
+ Transliterator.getInstance("NFD; [:Nonspacing Mark:] Remove"),
+ "Alphabētikós Katálogos",
+ "Alphabetikos Katalogos");
+ checkToken(Transliterator.getInstance("Han-Latin"), "中国", "zhōng guó");
}
-
+
public void testCustomFunctionality() throws Exception {
String rules = "a > b; b > c;"; // convert a's to b's and b's to c's
- checkToken(Transliterator.createFromRules("test", rules, Transliterator.FORWARD), "abacadaba", "bcbcbdbcb");
+ checkToken(
+ Transliterator.createFromRules("test", rules, Transliterator.FORWARD),
+ "abacadaba",
+ "bcbcbdbcb");
}
-
+
public void testCustomFunctionality2() throws Exception {
String rules = "c { a > b; a > d;"; // convert a's to b's and b's to c's
checkToken(Transliterator.createFromRules("test", rules, Transliterator.FORWARD), "caa", "cbd");
}
-
+
public void testOptimizer() throws Exception {
String rules = "a > b; b > c;"; // convert a's to b's and b's to c's
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
@@ -70,12 +65,11 @@ public class TestICUTransformFilter extends BaseTokenStreamTestCase {
new ICUTransformFilter(input, custom);
assertTrue(custom.getFilter().equals(new UnicodeSet("[ab]")));
}
-
+
public void testOptimizer2() throws Exception {
- checkToken(Transliterator.getInstance("Traditional-Simplified; CaseFold"),
- "ABCDE", "abcde");
+ checkToken(Transliterator.getInstance("Traditional-Simplified; CaseFold"), "ABCDE", "abcde");
}
-
+
public void testOptimizerSurrogate() throws Exception {
String rules = "\\U00020087 > x;"; // convert CJK UNIFIED IDEOGRAPH-20087 to an x
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
@@ -86,35 +80,41 @@ public class TestICUTransformFilter extends BaseTokenStreamTestCase {
assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]")));
}
- private void checkToken(Transliterator transform, String input, String expected) throws IOException {
+ private void checkToken(Transliterator transform, String input, String expected)
+ throws IOException {
final KeywordTokenizer input1 = new KeywordTokenizer();
input1.setReader(new StringReader(input));
TokenStream ts = new ICUTransformFilter(input1, transform);
- assertTokenStreamContents(ts, new String[] { expected });
+ assertTokenStreamContents(ts, new String[] {expected});
}
-
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
final Transliterator transform = Transliterator.getInstance("Any-Latin");
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, transform));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(
+ tokenizer, new ICUTransformFilter(tokenizer, transform));
+ }
+ };
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
a.close();
}
-
+
public void testEmptyTerm() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
- return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, Transliterator.getInstance("Any-Latin")));
- }
- };
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(
+ tokenizer,
+ new ICUTransformFilter(tokenizer, Transliterator.getInstance("Any-Latin")));
+ }
+ };
checkOneTerm(a, "", "");
a.close();
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java
index 10b9593..982c971 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java
@@ -16,61 +16,65 @@
*/
package org.apache.lucene.analysis.icu;
-
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
/** basic tests for {@link ICUTransformFilterFactory} */
public class TestICUTransformFilterFactory extends BaseTokenStreamTestCase {
-
+
/** ensure the transform is working */
public void test() throws Exception {
Reader reader = new StringReader("簡化字");
- Map<String,String> args = new HashMap<>();
+ Map<String, String> args = new HashMap<>();
args.put("id", "Traditional-Simplified");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "简化字" });
+ assertTokenStreamContents(stream, new String[] {"简化字"});
}
-
+
/** test forward and reverse direction */
public void testForwardDirection() throws Exception {
// forward
Reader reader = new StringReader("Российская Федерация");
- Map<String,String> args = new HashMap<>();
+ Map<String, String> args = new HashMap<>();
args.put("id", "Cyrillic-Latin");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "Rossijskaâ", "Federaciâ" });
+ assertTokenStreamContents(stream, new String[] {"Rossijskaâ", "Federaciâ"});
}
-
+
public void testReverseDirection() throws Exception {
// backward (invokes Latin-Cyrillic)
Reader reader = new StringReader("Rossijskaâ Federaciâ");
- Map<String,String> args = new HashMap<>();
+ Map<String, String> args = new HashMap<>();
args.put("id", "Cyrillic-Latin");
args.put("direction", "reverse");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] { "Российская", "Федерация" });
+ assertTokenStreamContents(stream, new String[] {"Российская", "Федерация"});
}
-
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new ICUTransformFilterFactory(new HashMap<String,String>() {{
- put("id", "Null");
- put("bogusArg", "bogusValue");
- }});
- });
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ new ICUTransformFilterFactory(
+ new HashMap<String, String>() {
+ {
+ put("id", "Null");
+ put("bogusArg", "bogusValue");
+ }
+ });
+ });
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestCharArrayIterator.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestCharArrayIterator.java
index 28856fb..49beec8 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestCharArrayIterator.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestCharArrayIterator.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import java.text.CharacterIterator;
-
import org.apache.lucene.util.LuceneTestCase;
public class TestCharArrayIterator extends LuceneTestCase {
@@ -35,24 +33,24 @@ public class TestCharArrayIterator extends LuceneTestCase {
assertEquals('t', ci.first());
assertEquals(CharacterIterator.DONE, ci.previous());
}
-
+
public void testFirst() {
CharArrayIterator ci = new CharArrayIterator();
ci.setText("testing".toCharArray(), 0, "testing".length());
ci.next();
- // Sets the position to getBeginIndex() and returns the character at that position.
+ // Sets the position to getBeginIndex() and returns the character at that position.
assertEquals('t', ci.first());
assertEquals(ci.getBeginIndex(), ci.getIndex());
// or DONE if the text is empty
ci.setText(new char[] {}, 0, 0);
assertEquals(CharacterIterator.DONE, ci.first());
}
-
+
public void testLast() {
CharArrayIterator ci = new CharArrayIterator();
ci.setText("testing".toCharArray(), 0, "testing".length());
- // Sets the position to getEndIndex()-1 (getEndIndex() if the text is empty)
- // and returns the character at that position.
+ // Sets the position to getEndIndex()-1 (getEndIndex() if the text is empty)
+ // and returns the character at that position.
assertEquals('g', ci.last());
assertEquals(ci.getIndex(), ci.getEndIndex() - 1);
// or DONE if the text is empty
@@ -60,10 +58,10 @@ public class TestCharArrayIterator extends LuceneTestCase {
assertEquals(CharacterIterator.DONE, ci.last());
assertEquals(ci.getEndIndex(), ci.getIndex());
}
-
+
public void testCurrent() {
CharArrayIterator ci = new CharArrayIterator();
- // Gets the character at the current position (as returned by getIndex()).
+ // Gets the character at the current position (as returned by getIndex()).
ci.setText("testing".toCharArray(), 0, "testing".length());
assertEquals('t', ci.current());
ci.last();
@@ -71,7 +69,7 @@ public class TestCharArrayIterator extends LuceneTestCase {
// or DONE if the current position is off the end of the text.
assertEquals(CharacterIterator.DONE, ci.current());
}
-
+
public void testNext() {
CharArrayIterator ci = new CharArrayIterator();
ci.setText("te".toCharArray(), 0, 2);
@@ -82,15 +80,17 @@ public class TestCharArrayIterator extends LuceneTestCase {
assertEquals(CharacterIterator.DONE, ci.next());
assertEquals(ci.getEndIndex(), ci.getIndex());
}
-
+
public void testSetIndex() {
CharArrayIterator ci = new CharArrayIterator();
ci.setText("test".toCharArray(), 0, "test".length());
- expectThrows(IllegalArgumentException.class, () -> {
- ci.setIndex(5);
- });
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ ci.setIndex(5);
+ });
}
-
+
public void testClone() {
char text[] = "testing".toCharArray();
CharArrayIterator ci = new CharArrayIterator();
@@ -101,6 +101,4 @@ public class TestCharArrayIterator extends LuceneTestCase {
assertEquals(ci.next(), ci2.next());
assertEquals(ci.last(), ci2.last());
}
-
-
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
index 0d9a49f..1dd5ca0 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
@@ -16,23 +16,21 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute;
-
import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.UnicodeSet;
-
import java.io.IOException;
import java.io.StringReader;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute;
public class TestICUTokenizer extends BaseTokenStreamTestCase {
-
+
public void testHugeDoc() throws IOException {
StringBuilder sb = new StringBuilder();
char whitespace[] = new char[4094];
@@ -40,45 +38,49 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase {
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
+ ICUTokenizer tokenizer =
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
tokenizer.setReader(new StringReader(input));
- assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
+ assertTokenStreamContents(tokenizer, new String[] {"testing", "1234"});
}
-
+
public void testHugeTerm2() throws IOException {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 40960; i++) {
sb.append('a');
}
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
+ ICUTokenizer tokenizer =
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
tokenizer.setReader(new StringReader(input));
char token[] = new char[4096];
Arrays.fill(token, 'a');
String expectedToken = new String(token);
- String expected[] = {
- expectedToken, expectedToken, expectedToken,
- expectedToken, expectedToken, expectedToken,
- expectedToken, expectedToken, expectedToken,
- expectedToken
+ String expected[] = {
+ expectedToken, expectedToken, expectedToken,
+ expectedToken, expectedToken, expectedToken,
+ expectedToken, expectedToken, expectedToken,
+ expectedToken
};
assertTokenStreamContents(tokenizer, expected);
}
-
- private Analyzer a;
-
+
+ private Analyzer a;
+
@Override
public void setUp() throws Exception {
super.setUp();
- a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
- return new TokenStreamComponents(tokenizer);
- }
- };
+ a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
+ return new TokenStreamComponents(tokenizer);
+ }
+ };
}
-
+
@Override
public void tearDown() throws Exception {
a.close();
@@ -86,262 +88,447 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase {
}
public void testArmenian() throws Exception {
- assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",
- new String[] { "Վիքիպեդիայի", "13", "միլիոն", "հոդվածները", "4,600", "հայերեն", "վիքիպեդիայում", "գրվել", "են", "կամավորների", "կողմից",
- "ու", "համարյա", "բոլոր", "հոդվածները", "կարող", "է", "խմբագրել", "ցանկաց", "մարդ", "ով", "կարող", "է", "բացել", "Վիքիպեդիայի", "կայքը" } );
+ assertAnalyzesTo(
+ a,
+ "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",
+ new String[] {
+ "Վիքիպեդիայի",
+ "13",
+ "միլիոն",
+ "հոդվածները",
+ "4,600",
+ "հայերեն",
+ "վիքիպեդիայում",
+ "գրվել",
+ "են",
+ "կամավորների",
+ "կողմից",
+ "ու",
+ "համարյա",
+ "բոլոր",
+ "հոդվածները",
+ "կարող",
+ "է",
+ "խմբագրել",
+ "ցանկաց",
+ "մարդ",
+ "ով",
+ "կարող",
+ "է",
+ "բացել",
+ "Վիքիպեդիայի",
+ "կայքը"
+ });
}
-
+
public void testAmharic() throws Exception {
- assertAnalyzesTo(a, "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
- new String[] { "ዊኪፔድያ", "የባለ", "ብዙ", "ቋንቋ", "የተሟላ", "ትክክለኛና", "ነጻ", "መዝገበ", "ዕውቀት", "ኢንሳይክሎፒዲያ", "ነው", "ማንኛውም" } );
+ assertAnalyzesTo(
+ a,
+ "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
+ new String[] {
+ "ዊኪፔድያ",
+ "የባለ",
+ "ብዙ",
+ "ቋንቋ",
+ "የተሟላ",
+ "ትክክለኛና",
+ "ነጻ",
+ "መዝገበ",
+ "ዕውቀት",
+ "ኢንሳይክሎፒዲያ",
+ "ነው",
+ "ማንኛውም"
+ });
}
-
+
public void testArabic() throws Exception {
- assertAnalyzesTo(a, "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",
- new String[] { "الفيلم", "الوثائقي", "الأول", "عن", "ويكيبيديا", "يسمى", "الحقيقة", "بالأرقام", "قصة", "ويكيبيديا",
- "بالإنجليزية", "Truth", "in", "Numbers", "The", "Wikipedia", "Story", "سيتم", "إطلاقه", "في", "2008" } );
+ assertAnalyzesTo(
+ a,
+ "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",
+ new String[] {
+ "الفيلم",
+ "الوثائقي",
+ "الأول",
+ "عن",
+ "ويكيبيديا",
+ "يسمى",
+ "الحقيقة",
+ "بالأرقام",
+ "قصة",
+ "ويكيبيديا",
+ "بالإنجليزية",
+ "Truth",
+ "in",
+ "Numbers",
+ "The",
+ "Wikipedia",
+ "Story",
+ "سيتم",
+ "إطلاقه",
+ "في",
+ "2008"
+ });
}
-
+
public void testAramaic() throws Exception {
- assertAnalyzesTo(a, "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",
- new String[] { "ܘܝܩܝܦܕܝܐ", "ܐܢܓܠܝܐ", "Wikipedia", "ܗܘ", "ܐܝܢܣܩܠܘܦܕܝܐ", "ܚܐܪܬܐ", "ܕܐܢܛܪܢܛ", "ܒܠܫܢ̈ܐ", "ܣܓܝܐ̈ܐ", "ܫܡܗ",
- "ܐܬܐ", "ܡܢ", "ܡ̈ܠܬܐ", "ܕ", "ܘܝܩܝ", "ܘ", "ܐܝܢܣܩܠܘܦܕܝܐ"});
+ assertAnalyzesTo(
+ a,
+ "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",
+ new String[] {
+ "ܘܝܩܝܦܕܝܐ",
+ "ܐܢܓܠܝܐ",
+ "Wikipedia",
+ "ܗܘ",
+ "ܐܝܢܣܩܠܘܦܕܝܐ",
+ "ܚܐܪܬܐ",
+ "ܕܐܢܛܪܢܛ",
+ "ܒܠܫܢ̈ܐ",
+ "ܣܓܝܐ̈ܐ",
+ "ܫܡܗ",
+ "ܐܬܐ",
+ "ܡܢ",
+ "ܡ̈ܠܬܐ",
+ "ܕ",
+ "ܘܝܩܝ",
+ "ܘ",
+ "ܐܝܢܣܩܠܘܦܕܝܐ"
+ });
}
-
+
public void testBengali() throws Exception {
- assertAnalyzesTo(a, "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
- new String[] { "এই", "বিশ্বকোষ", "পরিচালনা", "করে", "উইকিমিডিয়া", "ফাউন্ডেশন", "একটি", "অলাভজনক", "সংস্থা", "উইকিপিডিয়ার",
- "শুরু", "১৫", "জানুয়ারি", "২০০১", "সালে", "এখন", "পর্যন্ত", "২০০টিরও", "বেশী", "ভাষায়", "উইকিপিডিয়া", "রয়েছে" });
+ assertAnalyzesTo(
+ a,
+ "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
+ new String[] {
+ "এই",
+ "বিশ্বকোষ",
+ "পরিচালনা",
+ "করে",
+ "উইকিমিডিয়া",
+ "ফাউন্ডেশন",
+ "একটি",
+ "অলাভজনক",
+ "সংস্থা",
+ "উইকিপিডিয়ার",
+ "শুরু",
+ "১৫",
+ "জানুয়ারি",
+ "২০০১",
+ "সালে",
+ "এখন",
+ "পর্যন্ত",
+ "২০০টিরও",
+ "বেশী",
+ "ভাষায়",
+ "উইকিপিডিয়া",
+ "রয়েছে"
+ });
}
-
+
public void testFarsi() throws Exception {
- assertAnalyzesTo(a, "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",
- new String[] { "ویکی", "پدیای", "انگلیسی", "در", "تاریخ", "۲۵", "دی", "۱۳۷۹", "به", "صورت", "مکملی",
- "برای", "دانشنامهٔ", "تخصصی", "نوپدیا", "نوشته", "شد" });
+ assertAnalyzesTo(
+ a,
+ "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",
+ new String[] {
+ "ویکی",
+ "پدیای",
+ "انگلیسی",
+ "در",
+ "تاریخ",
+ "۲۵",
+ "دی",
+ "۱۳۷۹",
+ "به",
+ "صورت",
+ "مکملی",
+ "برای",
+ "دانشنامهٔ",
+ "تخصصی",
+ "نوپدیا",
+ "نوشته",
+ "شد"
+ });
}
-
+
public void testGreek() throws Exception {
- assertAnalyzesTo(a, "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",
- new String[] { "Γράφεται", "σε", "συνεργασία", "από", "εθελοντές", "με", "το", "λογισμικό", "wiki", "κάτι", "που",
- "σημαίνει", "ότι", "άρθρα", "μπορεί", "να", "προστεθούν", "ή", "να", "αλλάξουν", "από", "τον", "καθένα" });
+ assertAnalyzesTo(
+ a,
+ "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",
+ new String[] {
+ "Γράφεται",
+ "σε",
+ "συνεργασία",
+ "από",
+ "εθελοντές",
+ "με",
+ "το",
+ "λογισμικό",
+ "wiki",
+ "κάτι",
+ "που",
+ "σημαίνει",
+ "ότι",
+ "άρθρα",
+ "μπορεί",
+ "να",
+ "προστεθούν",
+ "ή",
+ "να",
+ "αλλάξουν",
+ "από",
+ "τον",
+ "καθένα"
+ });
}
-
+
public void testKhmer() throws Exception {
- assertAnalyzesTo(a, "ផ្ទះស្កឹមស្កៃបីបួនខ្នងនេះ", new String[] { "ផ្ទះ", "ស្កឹមស្កៃ", "បី", "បួន", "ខ្នង", "នេះ" });
+ assertAnalyzesTo(
+ a,
+ "ផ្ទះស្កឹមស្កៃបីបួនខ្នងនេះ",
+ new String[] {"ផ្ទះ", "ស្កឹមស្កៃ", "បី", "បួន", "ខ្នង", "នេះ"});
}
+
public void testLao() throws Exception {
- assertAnalyzesTo(a, "ກວ່າດອກ", new String[] { "ກວ່າ", "ດອກ" });
- assertAnalyzesTo(a, "ພາສາລາວ", new String[] { "ພາສາ", "ລາວ"}, new String[] { "<ALPHANUM>", "<ALPHANUM>" });
+ assertAnalyzesTo(a, "ກວ່າດອກ", new String[] {"ກວ່າ", "ດອກ"});
+ assertAnalyzesTo(
+ a, "ພາສາລາວ", new String[] {"ພາສາ", "ລາວ"}, new String[] {"<ALPHANUM>", "<ALPHANUM>"});
}
-
+
public void testMyanmar() throws Exception {
- assertAnalyzesTo(a, "သက်ဝင်လှုပ်ရှားစေပြီး", new String[] { "သက်ဝင်", "လှုပ်ရှား", "စေ", "ပြီး" });
+ assertAnalyzesTo(
+ a, "သက်ဝင်လှုပ်ရှားစေပြီး", new String[] {"သက်ဝင်", "လှုပ်ရှား", "စေ", "ပြီး"});
}
-
+
public void testThai() throws Exception {
- assertAnalyzesTo(a, "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "แล้ว", "เธอ", "จะ", "ไป", "ไหน", "๑๒๓๔"});
+ assertAnalyzesTo(
+ a,
+ "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",
+ new String[] {
+ "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "แล้ว", "เธอ", "จะ", "ไป", "ไหน",
+ "๑๒๓๔"
+ });
}
-
+
public void testTibetan() throws Exception {
- assertAnalyzesTo(a, "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
- new String[] { "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག", "མི", "ཉམས", "གོང", "འཕེལ", "དུ", "གཏོང", "བར", "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ" });
+ assertAnalyzesTo(
+ a,
+ "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
+ new String[] {
+ "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག", "མི", "ཉམས", "གོང", "འཕེལ", "དུ",
+ "གཏོང", "བར", "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ"
+ });
}
-
+
/*
* For chinese, tokenize as char (these can later form bigrams or whatever)
*/
public void testChinese() throws Exception {
- assertAnalyzesTo(a, "我是中国人。 １２３４ Ｔｅｓｔｓ ",
- new String[] { "我", "是", "中", "国", "人", "１２３４", "Ｔｅｓｔｓ"});
+ assertAnalyzesTo(
+ a, "我是中国人。 １２３４ Ｔｅｓｔｓ ", new String[] {"我", "是", "中", "国", "人", "１２３４", "Ｔｅｓｔｓ"});
}
-
+
public void testHebrew() throws Exception {
- assertAnalyzesTo(a, "דנקנר תקף את הדו\"ח",
- new String[] { "דנקנר", "תקף", "את", "הדו\"ח" });
- assertAnalyzesTo(a, "חברת בת של מודי'ס",
- new String[] { "חברת", "בת", "של", "מודי'ס" });
+ assertAnalyzesTo(a, "דנקנר תקף את הדו\"ח", new String[] {"דנקנר", "תקף", "את", "הדו\"ח"});
+ assertAnalyzesTo(a, "חברת בת של מודי'ס", new String[] {"חברת", "בת", "של", "מודי'ס"});
}
-
+
public void testEmpty() throws Exception {
assertAnalyzesTo(a, "", new String[] {});
assertAnalyzesTo(a, ".", new String[] {});
assertAnalyzesTo(a, " ", new String[] {});
}
-
+
/* test various jira issues this analyzer is related to */
-
+
public void testLUCENE1545() throws Exception {
/*
* Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTRE E.
* The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.
* Expected result is only on token "moͤchte".
*/
- assertAnalyzesTo(a, "moͤchte", new String[] { "moͤchte" });
+ assertAnalyzesTo(a, "moͤchte", new String[] {"moͤchte"});
}
-
+
/* Tests from StandardAnalyzer, just to show behavior is similar */
public void testAlphanumericSA() throws Exception {
// alphanumeric tokens
- assertAnalyzesTo(a, "B2B", new String[]{"B2B"});
- assertAnalyzesTo(a, "2B", new String[]{"2B"});
+ assertAnalyzesTo(a, "B2B", new String[] {"B2B"});
+ assertAnalyzesTo(a, "2B", new String[] {"2B"});
}
public void testDelimitersSA() throws Exception {
// other delimiters: "-", "/", ","
- assertAnalyzesTo(a, "some-dashed-phrase", new String[]{"some", "dashed", "phrase"});
- assertAnalyzesTo(a, "dogs,chase,cats", new String[]{"dogs", "chase", "cats"});
- assertAnalyzesTo(a, "ac/dc", new String[]{"ac", "dc"});
+ assertAnalyzesTo(a, "some-dashed-phrase", new String[] {"some", "dashed", "phrase"});
+ assertAnalyzesTo(a, "dogs,chase,cats", new String[] {"dogs", "chase", "cats"});
+ assertAnalyzesTo(a, "ac/dc", new String[] {"ac", "dc"});
}
public void testApostrophesSA() throws Exception {
// internal apostrophes: O'Reilly, you're, O'Reilly's
- assertAnalyzesTo(a, "O'Reilly", new String[]{"O'Reilly"});
- assertAnalyzesTo(a, "you're", new String[]{"you're"});
- assertAnalyzesTo(a, "she's", new String[]{"she's"});
- assertAnalyzesTo(a, "Jim's", new String[]{"Jim's"});
- assertAnalyzesTo(a, "don't", new String[]{"don't"});
- assertAnalyzesTo(a, "O'Reilly's", new String[]{"O'Reilly's"});
+ assertAnalyzesTo(a, "O'Reilly", new String[] {"O'Reilly"});
+ assertAnalyzesTo(a, "you're", new String[] {"you're"});
+ assertAnalyzesTo(a, "she's", new String[] {"she's"});
+ assertAnalyzesTo(a, "Jim's", new String[] {"Jim's"});
+ assertAnalyzesTo(a, "don't", new String[] {"don't"});
+ assertAnalyzesTo(a, "O'Reilly's", new String[] {"O'Reilly's"});
}
public void testNumericSA() throws Exception {
// floating point, serial, model numbers, ip addresses, etc.
// every other segment must have at least one digit
- assertAnalyzesTo(a, "21.35", new String[]{"21.35"});
- assertAnalyzesTo(a, "R2D2 C3PO", new String[]{"R2D2", "C3PO"});
- assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});
- assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});
+ assertAnalyzesTo(a, "21.35", new String[] {"21.35"});
+ assertAnalyzesTo(a, "R2D2 C3PO", new String[] {"R2D2", "C3PO"});
+ assertAnalyzesTo(a, "216.239.63.104", new String[] {"216.239.63.104"});
+ assertAnalyzesTo(a, "216.239.63.104", new String[] {"216.239.63.104"});
}
public void testTextWithNumbersSA() throws Exception {
// numbers
- assertAnalyzesTo(a, "David has 5000 bones", new String[]{"David", "has", "5000", "bones"});
+ assertAnalyzesTo(a, "David has 5000 bones", new String[] {"David", "has", "5000", "bones"});
}
public void testVariousTextSA() throws Exception {
// various
- assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"C", "embedded", "developers", "wanted"});
- assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "FOO", "BAR"});
- assertAnalyzesTo(a, "foo bar . FOO <> BAR", new String[]{"foo", "bar", "FOO", "BAR"});
- assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"QUOTED", "word"});
+ assertAnalyzesTo(
+ a, "C embedded developers wanted", new String[] {"C", "embedded", "developers", "wanted"});
+ assertAnalyzesTo(a, "foo bar FOO BAR", new String[] {"foo", "bar", "FOO", "BAR"});
+ assertAnalyzesTo(a, "foo bar . FOO <> BAR", new String[] {"foo", "bar", "FOO", "BAR"});
+ assertAnalyzesTo(a, "\"QUOTED\" word", new String[] {"QUOTED", "word"});
}
public void testKoreanSA() throws Exception {
// Korean words
- assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[]{"안녕하세요", "한글입니다"});
+ assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[] {"안녕하세요", "한글입니다"});
}
-
+
public void testReusableTokenStream() throws Exception {
- assertAnalyzesTo(a, "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
- new String[] { "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག", "མི", "ཉམས", "གོང",
- "འཕེལ", "དུ", "གཏོང", "བར", "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ" });
+ assertAnalyzesTo(
+ a,
+ "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
+ new String[] {
+ "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག", "མི", "ཉམས", "གོང", "འཕེལ", "དུ",
+ "གཏོང", "བར", "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ"
+ });
}
-
+
public void testOffsets() throws Exception {
- assertAnalyzesTo(a, "David has 5000 bones",
+ assertAnalyzesTo(
+ a,
+ "David has 5000 bones",
new String[] {"David", "has", "5000", "bones"},
new int[] {0, 6, 10, 15},
new int[] {5, 9, 14, 20});
}
-
+
public void testTypes() throws Exception {
- assertAnalyzesTo(a, "David has 5000 bones",
+ assertAnalyzesTo(
+ a,
+ "David has 5000 bones",
new String[] {"David", "has", "5000", "bones"},
- new String[] { "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>" });
+ new String[] {"<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>"});
}
-
+
public void testKorean() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "훈민정음",
- new String[] { "훈민정음" },
- new String[] { "<HANGUL>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "훈민정음", new String[] {"훈민정음"}, new String[] {"<HANGUL>"});
}
-
+
public void testJapanese() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "仮名遣い カタカナ",
- new String[] { "仮", "名", "遣", "い", "カタカナ" },
- new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ "仮名遣い カタカナ",
+ new String[] {"仮", "名", "遣", "い", "カタカナ"},
+ new String[] {
+ "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>"
+ });
}
-
+
/** simple emoji */
public void testEmoji() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "💩 💩💩",
- new String[] { "💩", "💩", "💩" },
- new String[] { "<EMOJI>", "<EMOJI>", "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ "💩 💩💩",
+ new String[] {"💩", "💩", "💩"},
+ new String[] {"<EMOJI>", "<EMOJI>", "<EMOJI>"});
}
-
+
/** emoji zwj sequence */
public void testEmojiSequence() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "👩❤️👩",
- new String[] { "👩❤️👩" },
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "👩❤️👩", new String[] {"👩❤️👩"}, new String[] {"<EMOJI>"});
}
-
+
/** emoji zwj sequence with fitzpatrick modifier */
public void testEmojiSequenceWithModifier() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "👨🏼⚕️",
- new String[] { "👨🏼⚕️" },
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "👨🏼⚕️", new String[] {"👨🏼⚕️"}, new String[] {"<EMOJI>"});
}
-
+
/** regional indicator */
public void testEmojiRegionalIndicator() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "🇺🇸🇺🇸",
- new String[] { "🇺🇸", "🇺🇸" },
- new String[] { "<EMOJI>", "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "🇺🇸🇺🇸", new String[] {"🇺🇸", "🇺🇸"}, new String[] {"<EMOJI>", "<EMOJI>"});
}
-
+
/** variation sequence */
public void testEmojiVariationSequence() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "#️⃣",
- new String[] { "#️⃣" },
- new String[] { "<EMOJI>" });
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "3️⃣",
- new String[] { "3️⃣",},
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "#️⃣", new String[] {"#️⃣"}, new String[] {"<EMOJI>"});
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ "3️⃣",
+ new String[] {
+ "3️⃣",
+ },
+ new String[] {"<EMOJI>"});
}
public void testEmojiTagSequence() throws Exception {
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "🏴",
- new String[] { "🏴" },
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, "🏴", new String[] {"🏴"}, new String[] {"<EMOJI>"});
}
-
+
public void testEmojiTokenization() throws Exception {
// simple emoji around latin
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "poo💩poo",
- new String[] { "poo", "💩", "poo" },
- new String[] { "<ALPHANUM>", "<EMOJI>", "<ALPHANUM>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ "poo💩poo",
+ new String[] {"poo", "💩", "poo"},
+ new String[] {"<ALPHANUM>", "<EMOJI>", "<ALPHANUM>"});
// simple emoji around non-latin
- BaseTokenStreamTestCase.assertAnalyzesTo(a, "💩中國💩",
- new String[] { "💩", "中", "國", "💩" },
- new String[] { "<EMOJI>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ "💩中國💩",
+ new String[] {"💩", "中", "國", "💩"},
+ new String[] {"<EMOJI>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<EMOJI>"});
}
-
+
public void testEmojiFromTheFuture() throws Exception {
// pick an unassigned character with extended_pictographic
int ch = new UnicodeSet("[[:Extended_Pictographic:]&[:Unassigned:]]").getRangeStart(0);
String value = new String(Character.toChars(ch));
// should analyze to emoji type
- BaseTokenStreamTestCase.assertAnalyzesTo(a, value,
- new String[] { value },
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a, value, new String[] {value}, new String[] {"<EMOJI>"});
// shouldn't break in a sequence
- BaseTokenStreamTestCase.assertAnalyzesTo(a, value + '\u200D' + value,
- new String[] { value + '\u200D' + value },
- new String[] { "<EMOJI>" });
+ BaseTokenStreamTestCase.assertAnalyzesTo(
+ a,
+ value + '\u200D' + value,
+ new String[] {value + '\u200D' + value},
+ new String[] {"<EMOJI>"});
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
}
-
+
/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
Random random = random();
checkRandomData(random, a, 10 * RANDOM_MULTIPLIER, 8192);
}
-
+
public void testTokenAttributes() throws Exception {
try (TokenStream ts = a.tokenStream("dummy", "This is a test")) {
ScriptAttribute scriptAtt = ts.addAttribute(ScriptAttribute.class);
@@ -355,38 +542,39 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase {
ts.end();
}
}
-
+
/** test for bugs like http://bugs.icu-project.org/trac/ticket/10767 */
public void testICUConcurrency() throws Exception {
int numThreads = 8;
final CountDownLatch startingGun = new CountDownLatch(1);
Thread threads[] = new Thread[numThreads];
for (int i = 0; i < threads.length; i++) {
- threads[i] = new Thread() {
- @Override
- public void run() {
- try {
- startingGun.await();
- long tokenCount = 0;
- final String contents = "英 เบียร์ ビール ເບຍ abc";
- for (int i = 0; i < 1000; i++) {
- try (Tokenizer tokenizer = new ICUTokenizer()) {
- tokenizer.setReader(new StringReader(contents));
- tokenizer.reset();
- while (tokenizer.incrementToken()) {
- tokenCount++;
+ threads[i] =
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ startingGun.await();
+ long tokenCount = 0;
+ final String contents = "英 เบียร์ ビール ເບຍ abc";
+ for (int i = 0; i < 1000; i++) {
+ try (Tokenizer tokenizer = new ICUTokenizer()) {
+ tokenizer.setReader(new StringReader(contents));
+ tokenizer.reset();
+ while (tokenizer.incrementToken()) {
+ tokenCount++;
+ }
+ tokenizer.end();
+ }
}
- tokenizer.end();
+ if (VERBOSE) {
+ System.out.println(tokenCount);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
}
- if (VERBOSE) {
- System.out.println(tokenCount);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
+ };
threads[i].start();
}
startingGun.countDown();
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
index 2612330..6d989b3 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
@@ -16,102 +16,92 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import java.util.Random;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-/**
- * test ICUTokenizer with dictionary-based CJ segmentation
- */
-@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8222")
+/** test ICUTokenizer with dictionary-based CJ segmentation */
+@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8222")
public class TestICUTokenizerCJK extends BaseTokenStreamTestCase {
Analyzer a;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
- a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true, true)));
- }
- };
+ a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true, true)));
+ }
+ };
}
-
+
@Override
public void tearDown() throws Exception {
a.close();
super.tearDown();
}
-
- /**
- * test stolen from smartcn
- */
+
+ /** test stolen from smartcn */
public void testSimpleChinese() throws Exception {
- assertAnalyzesTo(a, "我购买了道具和服装。",
- new String[] { "我", "购买", "了", "道具", "和", "服装" }
- );
+ assertAnalyzesTo(a, "我购买了道具和服装。", new String[] {"我", "购买", "了", "道具", "和", "服装"});
}
public void testTraditionalChinese() throws Exception {
- assertAnalyzesTo(a, "我購買了道具和服裝。",
- new String[] { "我", "購買", "了", "道具", "和", "服裝"});
- assertAnalyzesTo(a, "定義切分字串的基本單位是訂定分詞標準的首要工作", // From http://godel.iis.sinica.edu.tw/CKIP/paper/wordsegment_standard.pdf
- new String[] { "定義", "切", "分", "字串", "的", "基本", "單位", "是", "訂定", "分詞", "標準", "的", "首要", "工作" });
+ assertAnalyzesTo(a, "我購買了道具和服裝。", new String[] {"我", "購買", "了", "道具", "和", "服裝"});
+ assertAnalyzesTo(
+ a,
+ "定義切分字串的基本單位是訂定分詞標準的首要工作", // From
+ // http://godel.iis.sinica.edu.tw/CKIP/paper/wordsegment_standard.pdf
+ new String[] {
+ "定義", "切", "分", "字串", "的", "基本", "單位", "是", "訂定", "分詞", "標準", "的", "首要", "工作"
+ });
}
public void testChineseNumerics() throws Exception {
- assertAnalyzesTo(a, "９４８３", new String[] { "９４８３" });
- assertAnalyzesTo(a, "院內分機９４８３。",
- new String[] { "院", "內", "分機", "９４８３" });
- assertAnalyzesTo(a, "院內分機9483。",
- new String[] { "院", "內", "分機", "9483" });
+ assertAnalyzesTo(a, "９４８３", new String[] {"９４８３"});
+ assertAnalyzesTo(a, "院內分機９４８３。", new String[] {"院", "內", "分機", "９４８３"});
+ assertAnalyzesTo(a, "院內分機9483。", new String[] {"院", "內", "分機", "9483"});
}
-
- /**
- * test stolen from kuromoji
- */
+
+ /** test stolen from kuromoji */
public void testSimpleJapanese() throws Exception {
- assertAnalyzesTo(a, "それはまだ実験段階にあります",
- new String[] { "それ", "は", "まだ", "実験", "段階", "に", "あり", "ます" }
- );
+ assertAnalyzesTo(
+ a, "それはまだ実験段階にあります", new String[] {"それ", "は", "まだ", "実験", "段階", "に", "あり", "ます"});
}
-
- /**
- * dictionary segmentation with emoji
- */
+
+ /** dictionary segmentation with emoji */
public void testSimpleJapaneseWithEmoji() throws Exception {
- assertAnalyzesTo(a, "それはまだ実験段階にあります💩",
- new String[] { "それ", "は", "まだ", "実験", "段階", "に", "あり", "ます", "💩" }
- );
+ assertAnalyzesTo(
+ a, "それはまだ実験段階にあります💩", new String[] {"それ", "は", "まだ", "実験", "段階", "に", "あり", "ます", "💩"});
}
-
+
public void testJapaneseTypes() throws Exception {
- assertAnalyzesTo(a, "仮名遣い カタカナ",
- new String[] { "仮名遣い", "カタカナ" },
- new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>" });
+ assertAnalyzesTo(
+ a,
+ "仮名遣い カタカナ",
+ new String[] {"仮名遣い", "カタカナ"},
+ new String[] {"<IDEOGRAPHIC>", "<IDEOGRAPHIC>"});
}
-
+
public void testKorean() throws Exception {
// Korean words
- assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[]{"안녕하세요", "한글입니다"});
+ assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[] {"안녕하세요", "한글입니다"});
}
-
+
/** make sure that we still tag korean as HANGUL (for further decomposition/ngram/whatever) */
public void testKoreanTypes() throws Exception {
- assertAnalyzesTo(a, "훈민정음",
- new String[] { "훈민정음" },
- new String[] { "<HANGUL>" });
+ assertAnalyzesTo(a, "훈민정음", new String[] {"훈민정음"}, new String[] {"<HANGUL>"});
}
-
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 10000 * RANDOM_MULTIPLIER);
}
-
+
/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
Random random = random();
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
index 4f1e506..4ea9542 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
@@ -16,86 +16,125 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util.ClasspathResourceLoader;
-/** basic tests for {@link ICUTokenizerFactory} **/
+/** basic tests for {@link ICUTokenizerFactory} * */
public class TestICUTokenizerFactory extends BaseTokenStreamTestCase {
public void testMixedText() throws Exception {
Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี This is a test ກວ່າດອກ");
- ICUTokenizerFactory factory = new ICUTokenizerFactory(new HashMap<String,String>());
+ ICUTokenizerFactory factory = new ICUTokenizerFactory(new HashMap<String, String>());
factory.inform(new ClasspathResourceLoader(getClass()));
Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
- assertTokenStreamContents(stream,
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
- "This", "is", "a", "test", "ກວ່າ", "ດອກ"});
+ assertTokenStreamContents(
+ stream,
+ new String[] {
+ "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "This", "is", "a", "test",
+ "ກວ່າ", "ດອກ"
+ });
}
public void testTokenizeLatinOnWhitespaceOnly() throws Exception {
// “ U+201C LEFT DOUBLE QUOTATION MARK; ” U+201D RIGHT DOUBLE QUOTATION MARK
- Reader reader = new StringReader
- (" Don't,break.at?/(punct)! \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$ ");
- final Map<String,String> args = new HashMap<>();
+ Reader reader =
+ new StringReader(
+ " Don't,break.at?/(punct)! \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$ ");
+ final Map<String, String> args = new HashMap<>();
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(this.getClass()));
Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
- assertTokenStreamContents(stream,
- new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
- new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<OTHER>" });
+ assertTokenStreamContents(
+ stream,
+ new String[] {
+ "Don't,break.at?/(punct)!",
+ "\u201Cnice\u201D",
+ "85_At:all;",
+ "`really\"",
+ "+2=3$5,&813",
+ "!@#%$^)(*@#$"
+ },
+ new String[] {"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<OTHER>"});
}
public void testTokenizeLatinDontBreakOnHyphens() throws Exception {
- Reader reader = new StringReader
- ("One-two punch. Brang-, not brung-it. This one--not that one--is the right one, -ish.");
- final Map<String,String> args = new HashMap<>();
+ Reader reader =
+ new StringReader(
+ "One-two punch. Brang-, not brung-it. This one--not that one--is the right one, -ish.");
+ final Map<String, String> args = new HashMap<>();
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
- assertTokenStreamContents(stream,
- new String[] { "One-two", "punch",
- "Brang", "not", "brung-it",
- "This", "one", "not", "that", "one", "is", "the", "right", "one", "ish" });
+ assertTokenStreamContents(
+ stream,
+ new String[] {
+ "One-two",
+ "punch",
+ "Brang",
+ "not",
+ "brung-it",
+ "This",
+ "one",
+ "not",
+ "that",
+ "one",
+ "is",
+ "the",
+ "right",
+ "one",
+ "ish"
+ });
}
/**
- * Specify more than one script/rule file pair.
- * Override default DefaultICUTokenizerConfig Thai script tokenization.
- * Use the same rule file for both scripts.
+ * Specify more than one script/rule file pair. Override default DefaultICUTokenizerConfig Thai
+ * script tokenization. Use the same rule file for both scripts.
*/
public void testKeywordTokenizeCyrillicAndThai() throws Exception {
- Reader reader = new StringReader
- ("Some English. Немного русский. ข้อความภาษาไทยเล็ก ๆ น้อย ๆ More English.");
- final Map<String,String> args = new HashMap<>();
- args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
+ Reader reader =
+ new StringReader(
+ "Some English. Немного русский. ข้อความภาษาไทยเล็ก ๆ น้อย ๆ More English.");
+ final Map<String, String> args = new HashMap<>();
+ args.put(
+ ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
- assertTokenStreamContents(stream, new String[] { "Some", "English",
- "Немного русский. ",
- "ข้อความภาษาไทยเล็ก ๆ น้อย ๆ ",
- "More", "English" });
+ assertTokenStreamContents(
+ stream,
+ new String[] {
+ "Some",
+ "English",
+ "Немного русский. ",
+ "ข้อความภาษาไทยเล็ก ๆ น้อย ๆ ",
+ "More",
+ "English"
+ });
}
-
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new ICUTokenizerFactory(new HashMap<String,String>() {{
- put("bogusArg", "bogusValue");
- }});
- });
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ new ICUTokenizerFactory(
+ new HashMap<String, String>() {
+ {
+ put("bogusArg", "bogusValue");
+ }
+ });
+ });
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java
index a3b608e..33bfa01 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java
@@ -28,129 +28,135 @@ public class TestMyanmarSyllable extends BaseTokenStreamTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
- a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, false));
- return new TokenStreamComponents(tokenizer);
- }
- };
- }
-
+ a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new ICUTokenizer(
+ newAttributeFactory(), new DefaultICUTokenizerConfig(false, false));
+ return new TokenStreamComponents(tokenizer);
+ }
+ };
+ }
+
@Override
public void tearDown() throws Exception {
a.close();
super.tearDown();
}
-
+
/** as opposed to dictionary break of သက်ဝင်|လှုပ်ရှား|စေ|ပြီး */
public void testBasics() throws Exception {
- assertAnalyzesTo(a, "သက်ဝင်လှုပ်ရှားစေပြီး", new String[] { "သက်", "ဝင်", "လှုပ်", "ရှား", "စေ", "ပြီး" });
+ assertAnalyzesTo(
+ a, "သက်ဝင်လှုပ်ရှားစေပြီး", new String[] {"သက်", "ဝင်", "လှုပ်", "ရှား", "စေ", "ပြီး"});
}
-
- // simple tests from "A Rule-based Syllable Segmentation of Myanmar Text"
+
+ // simple tests from "A Rule-based Syllable Segmentation of Myanmar Text"
// * http://www.aclweb.org/anthology/I08-3010
- // (see also the presentation: http://gii2.nagaokaut.ac.jp/gii/media/share/20080901-ZMM%20Presentation.pdf)
+ // (see also the presentation:
+ // http://gii2.nagaokaut.ac.jp/gii/media/share/20080901-ZMM%20Presentation.pdf)
// The words are fake, we just test the categories.
- // note that currently our algorithm is not sophisticated enough to handle some of the special cases!
-
+ // note that currently our algorithm is not sophisticated enough to handle some of the special
+ // cases!
+
/** constant */
public void testC() throws Exception {
- assertAnalyzesTo(a, "ကက", new String[] { "က", "က" });
+ assertAnalyzesTo(a, "ကက", new String[] {"က", "က"});
}
-
+
/** consonant + sign */
public void testCF() throws Exception {
- assertAnalyzesTo(a, "ကံကံ", new String[] { "ကံ", "ကံ" });
+ assertAnalyzesTo(a, "ကံကံ", new String[] {"ကံ", "ကံ"});
}
-
+
/** consonant + consonant + asat */
public void testCCA() throws Exception {
- assertAnalyzesTo(a, "ကင်ကင်", new String[] { "ကင်", "ကင်" });
+ assertAnalyzesTo(a, "ကင်ကင်", new String[] {"ကင်", "ကင်"});
}
-
+
/** consonant + consonant + asat + sign */
public void testCCAF() throws Exception {
- assertAnalyzesTo(a, "ကင်းကင်း", new String[] { "ကင်း", "ကင်း" });
+ assertAnalyzesTo(a, "ကင်းကင်း", new String[] {"ကင်း", "ကင်း"});
}
-
+
/** consonant + vowel */
public void testCV() throws Exception {
- assertAnalyzesTo(a, "ကာကာ", new String[] { "ကာ", "ကာ" });
+ assertAnalyzesTo(a, "ကာကာ", new String[] {"ကာ", "ကာ"});
}
-
+
/** consonant + vowel + sign */
public void testCVF() throws Exception {
- assertAnalyzesTo(a, "ကားကား", new String[] { "ကား", "ကား" });
+ assertAnalyzesTo(a, "ကားကား", new String[] {"ကား", "ကား"});
}
-
+
/** consonant + vowel + vowel + asat */
public void testCVVA() throws Exception {
- assertAnalyzesTo(a, "ကော်ကော်", new String[] { "ကော်", "ကော်" });
+ assertAnalyzesTo(a, "ကော်ကော်", new String[] {"ကော်", "ကော်"});
}
-
+
/** consonant + vowel + vowel + consonant + asat */
public void testCVVCA() throws Exception {
- assertAnalyzesTo(a, "ကောင်ကောင်", new String[] { "ကောင်", "ကောင်" });
+ assertAnalyzesTo(a, "ကောင်ကောင်", new String[] {"ကောင်", "ကောင်"});
}
-
+
/** consonant + vowel + vowel + consonant + asat + sign */
public void testCVVCAF() throws Exception {
- assertAnalyzesTo(a, "ကောင်းကောင်း", new String[] { "ကောင်း", "ကောင်း" });
+ assertAnalyzesTo(a, "ကောင်းကောင်း", new String[] {"ကောင်း", "ကောင်း"});
}
-
+
/** consonant + medial */
public void testCM() throws Exception {
- assertAnalyzesTo(a, "ကျကျ", new String[] { "ကျ", "ကျ" });
+ assertAnalyzesTo(a, "ကျကျ", new String[] {"ကျ", "ကျ"});
}
-
+
/** consonant + medial + sign */
public void testCMF() throws Exception {
- assertAnalyzesTo(a, "ကျံကျံ", new String[] { "ကျံ", "ကျံ" });
+ assertAnalyzesTo(a, "ကျံကျံ", new String[] {"ကျံ", "ကျံ"});
}
-
+
/** consonant + medial + consonant + asat */
public void testCMCA() throws Exception {
- assertAnalyzesTo(a, "ကျင်ကျင်", new String[] { "ကျင်", "ကျင်" });
+ assertAnalyzesTo(a, "ကျင်ကျင်", new String[] {"ကျင်", "ကျင်"});
}
-
+
/** consonant + medial + consonant + asat + sign */
public void testCMCAF() throws Exception {
- assertAnalyzesTo(a, "ကျင်းကျင်း", new String[] { "ကျင်း", "ကျင်း" });
+ assertAnalyzesTo(a, "ကျင်းကျင်း", new String[] {"ကျင်း", "ကျင်း"});
}
-
+
/** consonant + medial + vowel */
public void testCMV() throws Exception {
- assertAnalyzesTo(a, "ကျာကျာ", new String[] { "ကျာ", "ကျာ" });
+ assertAnalyzesTo(a, "ကျာကျာ", new String[] {"ကျာ", "ကျာ"});
}
-
+
/** consonant + medial + vowel + sign */
public void testCMVF() throws Exception {
- assertAnalyzesTo(a, "ကျားကျား", new String[] { "ကျား", "ကျား" });
+ assertAnalyzesTo(a, "ကျားကျား", new String[] {"ကျား", "ကျား"});
}
-
+
/** consonant + medial + vowel + vowel + asat */
public void testCMVVA() throws Exception {
- assertAnalyzesTo(a, "ကျော်ကျော်", new String[] { "ကျော်", "ကျော်" });
+ assertAnalyzesTo(a, "ကျော်ကျော်", new String[] {"ကျော်", "ကျော်"});
}
-
+
/** consonant + medial + vowel + vowel + consonant + asat */
public void testCMVVCA() throws Exception {
- assertAnalyzesTo(a, "ကြောင်ကြောင်", new String[] { "ကြောင်", "ကြောင်"});
+ assertAnalyzesTo(a, "ကြောင်ကြောင်", new String[] {"ကြောင်", "ကြောင်"});
}
-
+
/** consonant + medial + vowel + vowel + consonant + asat + sign */
public void testCMVVCAF() throws Exception {
- assertAnalyzesTo(a, "ကြောင်းကြောင်း", new String[] { "ကြောင်း", "ကြောင်း"});
+ assertAnalyzesTo(a, "ကြောင်းကြောင်း", new String[] {"ကြောင်း", "ကြောင်း"});
}
-
+
/** independent vowel */
public void testI() throws Exception {
- assertAnalyzesTo(a, "ဪဪ", new String[] { "ဪ", "ဪ" });
+ assertAnalyzesTo(a, "ဪဪ", new String[] {"ဪ", "ဪ"});
}
-
+
/** independent vowel */
public void testE() throws Exception {
- assertAnalyzesTo(a, "ဣဣ", new String[] { "ဣ", "ဣ" });
+ assertAnalyzesTo(a, "ဣဣ", new String[] {"ဣ", "ဣ"});
}
}
diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
index 411b85e..7e053ae 100644
--- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
+++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.icu.segmentation;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharArraySet;
@@ -30,208 +28,309 @@ import org.apache.lucene.analysis.icu.ICUNormalizer2Filter;
import org.apache.lucene.util.IOUtils;
/**
- * Tests ICUTokenizer's ability to work with CJKBigramFilter.
- * Most tests adopted from TestCJKTokenizer
+ * Tests ICUTokenizer's ability to work with CJKBigramFilter. Most tests adopted from
+ * TestCJKTokenizer
*/
public class TestWithCJKBigramFilter extends BaseTokenStreamTestCase {
-
+
Analyzer analyzer, analyzer2;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
/*
* ICUTokenizer+CJKBigramFilter
*/
- analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
- TokenStream result = new CJKBigramFilter(source);
- return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
- }
- };
+ analyzer =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source =
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
+ TokenStream result = new CJKBigramFilter(source);
+ return new TokenStreamComponents(
+ source, new StopFilter(result, CharArraySet.EMPTY_SET));
+ }
+ };
/*
* ICUTokenizer+ICUNormalizer2Filter+CJKBigramFilter.
- *
+ *
* ICUNormalizer2Filter uses nfkc_casefold by default, so this is a language-independent
* superset of CJKWidthFilter's foldings.
*/
- analyzer2 = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
- // we put this before the CJKBigramFilter, because the normalization might combine
- // some halfwidth katakana forms, which will affect the bigramming.
- TokenStream result = new ICUNormalizer2Filter(source);
- result = new CJKBigramFilter(result);
- return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
- }
- };
+ analyzer2 =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source =
+ new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true));
+ // we put this before the CJKBigramFilter, because the normalization might combine
+ // some halfwidth katakana forms, which will affect the bigramming.
+ TokenStream result = new ICUNormalizer2Filter(source);
+ result = new CJKBigramFilter(result);
+ return new TokenStreamComponents(
+ source, new StopFilter(result, CharArraySet.EMPTY_SET));
+ }
+ };
}
-
+
@Override
public void tearDown() throws Exception {
IOUtils.close(analyzer, analyzer2);
super.tearDown();
}
-
+
public void testJa1() throws IOException {
- assertAnalyzesTo(analyzer, "一二三四五六七八九十",
- new String[] { "一二", "二三", "三四", "四五", "五六", "六七", "七八", "八九", "九十" },
- new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 },
- new int[] { 2, 3, 4, 5, 6, 7, 8, 9, 10 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一二三四五六七八九十",
+ new String[] {"一二", "二三", "三四", "四五", "五六", "六七", "七八", "八九", "九十"},
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8},
+ new int[] {2, 3, 4, 5, 6, 7, 8, 9, 10},
+ new String[] {
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1});
}
-
+
public void testJa2() throws IOException {
- assertAnalyzesTo(analyzer, "一 二三四 五六七八九 十",
- new String[] { "一", "二三", "三四", "五六", "六七", "七八", "八九", "十" },
- new int[] { 0, 2, 3, 6, 7, 8, 9, 12 },
- new int[] { 1, 4, 5, 8, 9, 10, 11, 13 },
- new String[] { "<SINGLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<SINGLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一 二三四 五六七八九 十",
+ new String[] {"一", "二三", "三四", "五六", "六七", "七八", "八九", "十"},
+ new int[] {0, 2, 3, 6, 7, 8, 9, 12},
+ new int[] {1, 4, 5, 8, 9, 10, 11, 13},
+ new String[] {
+ "<SINGLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<SINGLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1});
}
-
+
public void testC() throws IOException {
- assertAnalyzesTo(analyzer, "abc defgh ijklmn opqrstu vwxy z",
- new String[] { "abc", "defgh", "ijklmn", "opqrstu", "vwxy", "z" },
- new int[] { 0, 4, 10, 17, 25, 30 },
- new int[] { 3, 9, 16, 24, 29, 31 },
- new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>" },
- new int[] { 1, 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "abc defgh ijklmn opqrstu vwxy z",
+ new String[] {"abc", "defgh", "ijklmn", "opqrstu", "vwxy", "z"},
+ new int[] {0, 4, 10, 17, 25, 30},
+ new int[] {3, 9, 16, 24, 29, 31},
+ new String[] {
+ "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1});
}
-
- /**
- * LUCENE-2207: wrong offset calculated by end()
- */
+
+ /** LUCENE-2207: wrong offset calculated by end() */
public void testFinalOffset() throws IOException {
- assertAnalyzesTo(analyzer, "あい",
- new String[] { "あい" },
- new int[] { 0 },
- new int[] { 2 },
- new String[] { "<DOUBLE>" },
- new int[] { 1 });
-
- assertAnalyzesTo(analyzer, "あい ",
- new String[] { "あい" },
- new int[] { 0 },
- new int[] { 2 },
- new String[] { "<DOUBLE>" },
- new int[] { 1 });
-
- assertAnalyzesTo(analyzer, "test",
- new String[] { "test" },
- new int[] { 0 },
- new int[] { 4 },
- new String[] { "<ALPHANUM>" },
- new int[] { 1 });
-
- assertAnalyzesTo(analyzer, "test ",
- new String[] { "test" },
- new int[] { 0 },
- new int[] { 4 },
- new String[] { "<ALPHANUM>" },
- new int[] { 1 });
-
- assertAnalyzesTo(analyzer, "あいtest",
- new String[] { "あい", "test" },
- new int[] { 0, 2 },
- new int[] { 2, 6 },
- new String[] { "<DOUBLE>", "<ALPHANUM>" },
- new int[] { 1, 1 });
-
- assertAnalyzesTo(analyzer, "testあい ",
- new String[] { "test", "あい" },
- new int[] { 0, 4 },
- new int[] { 4, 6 },
- new String[] { "<ALPHANUM>", "<DOUBLE>" },
- new int[] { 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "あい",
+ new String[] {"あい"},
+ new int[] {0},
+ new int[] {2},
+ new String[] {"<DOUBLE>"},
+ new int[] {1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "あい ",
+ new String[] {"あい"},
+ new int[] {0},
+ new int[] {2},
+ new String[] {"<DOUBLE>"},
+ new int[] {1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "test",
+ new String[] {"test"},
+ new int[] {0},
+ new int[] {4},
+ new String[] {"<ALPHANUM>"},
+ new int[] {1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "test ",
+ new String[] {"test"},
+ new int[] {0},
+ new int[] {4},
+ new String[] {"<ALPHANUM>"},
+ new int[] {1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "あいtest",
+ new String[] {"あい", "test"},
+ new int[] {0, 2},
+ new int[] {2, 6},
+ new String[] {"<DOUBLE>", "<ALPHANUM>"},
+ new int[] {1, 1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "testあい ",
+ new String[] {"test", "あい"},
+ new int[] {0, 4},
+ new int[] {4, 6},
+ new String[] {"<ALPHANUM>", "<DOUBLE>"},
+ new int[] {1, 1});
}
-
+
public void testMix() throws IOException {
- assertAnalyzesTo(analyzer, "あいうえおabcかきくけこ",
- new String[] { "あい", "いう", "うえ", "えお", "abc", "かき", "きく", "くけ", "けこ" },
- new int[] { 0, 1, 2, 3, 5, 8, 9, 10, 11 },
- new int[] { 2, 3, 4, 5, 8, 10, 11, 12, 13 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<ALPHANUM>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1});
+ assertAnalyzesTo(
+ analyzer,
+ "あいうえおabcかきくけこ",
+ new String[] {"あい", "いう", "うえ", "えお", "abc", "かき", "きく", "くけ", "けこ"},
+ new int[] {0, 1, 2, 3, 5, 8, 9, 10, 11},
+ new int[] {2, 3, 4, 5, 8, 10, 11, 12, 13},
+ new String[] {
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<ALPHANUM>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1});
}
-
+
public void testMix2() throws IOException {
- assertAnalyzesTo(analyzer, "あいうえおabんcかきくけ こ",
- new String[] { "あい", "いう", "うえ", "えお", "ab", "ん", "c", "かき", "きく", "くけ", "こ" },
- new int[] { 0, 1, 2, 3, 5, 7, 8, 9, 10, 11, 14 },
- new int[] { 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<ALPHANUM>", "<SINGLE>", "<ALPHANUM>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<SINGLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "あいうえおabんcかきくけ こ",
+ new String[] {"あい", "いう", "うえ", "えお", "ab", "ん", "c", "かき", "きく", "くけ", "こ"},
+ new int[] {0, 1, 2, 3, 5, 7, 8, 9, 10, 11, 14},
+ new int[] {2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15},
+ new String[] {
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<ALPHANUM>",
+ "<SINGLE>",
+ "<ALPHANUM>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<SINGLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1});
}
-
- /**
- * Non-english text (outside of CJK) is treated normally, according to unicode rules
- */
+
+ /** Non-english text (outside of CJK) is treated normally, according to unicode rules */
public void testNonIdeographic() throws IOException {
- assertAnalyzesTo(analyzer, "一 روبرت موير",
- new String[] { "一", "روبرت", "موير" },
- new int[] { 0, 2, 8 },
- new int[] { 1, 7, 12 },
- new String[] { "<SINGLE>", "<ALPHANUM>", "<ALPHANUM>" },
- new int[] { 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一 روبرت موير",
+ new String[] {"一", "روبرت", "موير"},
+ new int[] {0, 2, 8},
+ new int[] {1, 7, 12},
+ new String[] {"<SINGLE>", "<ALPHANUM>", "<ALPHANUM>"},
+ new int[] {1, 1, 1});
}
-
- /**
- * Same as the above, except with a nonspacing mark to show correctness.
- */
+
+ /** Same as the above, except with a nonspacing mark to show correctness. */
public void testNonIdeographicNonLetter() throws IOException {
- assertAnalyzesTo(analyzer, "一 رُوبرت موير",
- new String[] { "一", "رُوبرت", "موير" },
- new int[] { 0, 2, 9 },
- new int[] { 1, 8, 13 },
- new String[] { "<SINGLE>", "<ALPHANUM>", "<ALPHANUM>" },
- new int[] { 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一 رُوبرت موير",
+ new String[] {"一", "رُوبرت", "موير"},
+ new int[] {0, 2, 9},
+ new int[] {1, 8, 13},
+ new String[] {"<SINGLE>", "<ALPHANUM>", "<ALPHANUM>"},
+ new int[] {1, 1, 1});
}
-
+
public void testSurrogates() throws IOException {
- assertAnalyzesTo(analyzer, "𩬅艱鍟䇹愯瀛",
- new String[] { "𩬅艱", "艱鍟", "鍟䇹", "䇹愯", "愯瀛" },
- new int[] { 0, 2, 3, 4, 5 },
- new int[] { 3, 4, 5, 6, 7 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>" },
- new int[] { 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "𩬅艱鍟䇹愯瀛",
+ new String[] {"𩬅艱", "艱鍟", "鍟䇹", "䇹愯", "愯瀛"},
+ new int[] {0, 2, 3, 4, 5},
+ new int[] {3, 4, 5, 6, 7},
+ new String[] {"<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>"},
+ new int[] {1, 1, 1, 1, 1});
}
-
+
public void testReusableTokenStream() throws IOException {
- assertAnalyzesTo(analyzer, "あいうえおabcかきくけこ",
- new String[] { "あい", "いう", "うえ", "えお", "abc", "かき", "きく", "くけ", "けこ" },
- new int[] { 0, 1, 2, 3, 5, 8, 9, 10, 11 },
- new int[] { 2, 3, 4, 5, 8, 10, 11, 12, 13 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<ALPHANUM>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1});
-
- assertAnalyzesTo(analyzer, "あいうえおabんcかきくけ こ",
- new String[] { "あい", "いう", "うえ", "えお", "ab", "ん", "c", "かき", "きく", "くけ", "こ" },
- new int[] { 0, 1, 2, 3, 5, 7, 8, 9, 10, 11, 14 },
- new int[] { 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15 },
- new String[] { "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<ALPHANUM>", "<SINGLE>", "<ALPHANUM>", "<DOUBLE>", "<DOUBLE>", "<DOUBLE>", "<SINGLE>" },
- new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "あいうえおabcかきくけこ",
+ new String[] {"あい", "いう", "うえ", "えお", "abc", "かき", "きく", "くけ", "けこ"},
+ new int[] {0, 1, 2, 3, 5, 8, 9, 10, 11},
+ new int[] {2, 3, 4, 5, 8, 10, 11, 12, 13},
+ new String[] {
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<ALPHANUM>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1});
+
+ assertAnalyzesTo(
+ analyzer,
+ "あいうえおabんcかきくけ こ",
+ new String[] {"あい", "いう", "うえ", "えお", "ab", "ん", "c", "かき", "きく", "くけ", "こ"},
+ new int[] {0, 1, 2, 3, 5, 7, 8, 9, 10, 11, 14},
+ new int[] {2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15},
+ new String[] {
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<ALPHANUM>",
+ "<SINGLE>",
+ "<ALPHANUM>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<DOUBLE>",
+ "<SINGLE>"
+ },
+ new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1});
}
-
+
public void testSingleChar() throws IOException {
- assertAnalyzesTo(analyzer, "一",
- new String[] { "一" },
- new int[] { 0 },
- new int[] { 1 },
- new String[] { "<SINGLE>" },
- new int[] { 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一",
+ new String[] {"一"},
+ new int[] {0},
+ new int[] {1},
+ new String[] {"<SINGLE>"},
+ new int[] {1});
}
-
+
public void testTokenStream() throws IOException {
- assertAnalyzesTo(analyzer, "一丁丂",
- new String[] { "一丁", "丁丂"},
- new int[] { 0, 1 },
- new int[] { 2, 3 },
- new String[] { "<DOUBLE>", "<DOUBLE>" },
- new int[] { 1, 1 });
+ assertAnalyzesTo(
+ analyzer,
+ "一丁丂",
+ new String[] {"一丁", "丁丂"},
+ new int[] {0, 1},
+ new int[] {2, 3},
+ new String[] {"<DOUBLE>", "<DOUBLE>"},
+ new int[] {1, 1});
}
}
diff --git a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
index b8d8046..18818a3 100644
--- a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
+++ b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
@@ -16,12 +16,10 @@
*/
package org.apache.lucene.analysis.icu;
-
import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.lang.UProperty;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.text.UnicodeSetIterator;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
@@ -45,19 +43,14 @@ import java.util.regex.Pattern;
/**
* Downloads/generates lucene/analysis/icu/src/data/utr30/*.txt
*
- * ASSUMPTION: This class will be run with current directory set to
+ * <p>ASSUMPTION: This class will be run with current directory set to
* lucene/analysis/icu/src/data/utr30/
*
* <ol>
- * <li>
- * Downloads nfc.txt, nfkc.txt and nfkc_cf.txt from icu-project.org,
- * overwriting the versions in lucene/analysis/icu/src/data/utr30/.
- * </li>
- * <li>
- * Converts round-trip mappings in nfc.txt (containing '=')
- * that map to at least one [:Diacritic:] character
- * into one-way mappings ('>' instead of '=').
- * </li>
+ * <li>Downloads nfc.txt, nfkc.txt and nfkc_cf.txt from icu-project.org, overwriting the versions
+ * in lucene/analysis/icu/src/data/utr30/.
+ * <li>Converts round-trip mappings in nfc.txt (containing '=') that map to at least one
+ * [:Diacritic:] character into one-way mappings ('>' instead of '=').
* </ol>
*/
public class GenerateUTR30DataFiles {
@@ -68,16 +61,15 @@ public class GenerateUTR30DataFiles {
private static final String NFKC_TXT = "nfkc.txt";
private static final String NFKC_CF_TXT = "nfkc_cf.txt";
private static byte[] DOWNLOAD_BUFFER = new byte[8192];
- private static final Pattern ROUND_TRIP_MAPPING_LINE_PATTERN
- = Pattern.compile("^\\s*([^=]+?)\\s*=\\s*(.*)$");
- private static final Pattern VERBATIM_RULE_LINE_PATTERN
- = Pattern.compile("^#\\s*Rule:\\s*verbatim\\s*$", Pattern.CASE_INSENSITIVE);
- private static final Pattern RULE_LINE_PATTERN
- = Pattern.compile("^#\\s*Rule:\\s*(.*)>(.*)", Pattern.CASE_INSENSITIVE);
- private static final Pattern BLANK_OR_COMMENT_LINE_PATTERN
- = Pattern.compile("^\\s*(?:#.*)?$");
- private static final Pattern NUMERIC_VALUE_PATTERN
- = Pattern.compile("Numeric[-\\s_]*Value", Pattern.CASE_INSENSITIVE);
+ private static final Pattern ROUND_TRIP_MAPPING_LINE_PATTERN =
+ Pattern.compile("^\\s*([^=]+?)\\s*=\\s*(.*)$");
+ private static final Pattern VERBATIM_RULE_LINE_PATTERN =
+ Pattern.compile("^#\\s*Rule:\\s*verbatim\\s*$", Pattern.CASE_INSENSITIVE);
+ private static final Pattern RULE_LINE_PATTERN =
+ Pattern.compile("^#\\s*Rule:\\s*(.*)>(.*)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern BLANK_OR_COMMENT_LINE_PATTERN = Pattern.compile("^\\s*(?:#.*)?$");
+ private static final Pattern NUMERIC_VALUE_PATTERN =
+ Pattern.compile("Numeric[-\\s_]*Value", Pattern.CASE_INSENSITIVE);
public static void main(String args[]) {
try {
@@ -90,15 +82,18 @@ public class GenerateUTR30DataFiles {
}
private static void expandRulesInUTR30DataFiles() throws IOException {
- FileFilter filter = new FileFilter() {
- @Override
- public boolean accept(File pathname) {
- String name = pathname.getName();
- return pathname.isFile() && name.matches(".*\\.(?s:txt)")
- && ! name.equals(NFC_TXT) && ! name.equals(NFKC_TXT)
- && ! name.equals(NFKC_CF_TXT);
- }
- };
+ FileFilter filter =
+ new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ String name = pathname.getName();
+ return pathname.isFile()
+ && name.matches(".*\\.(?s:txt)")
+ && !name.equals(NFC_TXT)
+ && !name.equals(NFKC_TXT)
+ && !name.equals(NFKC_CF_TXT);
+ }
+ };
for (File file : new File(".").listFiles(filter)) {
expandDataFileRules(file);
}
@@ -129,8 +124,7 @@ public class GenerateUTR30DataFiles {
String rightHandSide = ruleMatcher.group(2).trim();
expandSingleRule(builder, leftHandSide, rightHandSide);
} catch (IllegalArgumentException e) {
- System.err.println
- ("ERROR in " + file.getName() + " line #" + lineNum + ":");
+ System.err.println("ERROR in " + file.getName() + " line #" + lineNum + ":");
e.printStackTrace(System.err);
System.exit(1);
}
@@ -177,8 +171,9 @@ public class GenerateUTR30DataFiles {
System.err.print("Downloading " + NFKC_CF_TXT + " and making diacritic rules one-way ... ");
URLConnection connection = openConnection(new URL(norm2url, NFC_TXT));
- BufferedReader reader = new BufferedReader
- (new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
+ BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
Writer writer = new OutputStreamWriter(new FileOutputStream(NFC_TXT), StandardCharsets.UTF_8);
try {
String line;
@@ -197,7 +192,7 @@ public class GenerateUTR30DataFiles {
diacritics.add(outputCodePoint);
}
}
- if ( ! diacritics.isEmpty()) {
+ if (!diacritics.isEmpty()) {
StringBuilder replacementLine = new StringBuilder();
replacementLine.append(leftHandSide).append(">").append(rightHandSide);
replacementLine.append(" # one-way: diacritic");
@@ -220,8 +215,7 @@ public class GenerateUTR30DataFiles {
System.err.println("done.");
}
- private static void download(URL url, String outputFile)
- throws IOException {
+ private static void download(URL url, String outputFile) throws IOException {
final URLConnection connection = openConnection(url);
final InputStream inputStream = connection.getInputStream();
final OutputStream outputStream = new FileOutputStream(outputFile);
@@ -244,17 +238,18 @@ public class GenerateUTR30DataFiles {
return connection;
}
- private static void expandSingleRule
- (StringBuilder builder, String leftHandSide, String rightHandSide)
+ private static void expandSingleRule(
+ StringBuilder builder, String leftHandSide, String rightHandSide)
throws IllegalArgumentException {
UnicodeSet set = new UnicodeSet(leftHandSide, UnicodeSet.IGNORE_SPACE);
boolean numericValue = NUMERIC_VALUE_PATTERN.matcher(rightHandSide).matches();
- for (UnicodeSetIterator it = new UnicodeSetIterator(set) ; it.nextRange() ; ) {
+ for (UnicodeSetIterator it = new UnicodeSetIterator(set); it.nextRange(); ) {
if (it.codepoint != UnicodeSetIterator.IS_STRING) {
if (numericValue) {
- for (int cp = it.codepoint ; cp <= it.codepointEnd ; ++cp) {
+ for (int cp = it.codepoint; cp <= it.codepointEnd; ++cp) {
builder.append(String.format(Locale.ROOT, "%04X", cp)).append('>');
- builder.append(String.format(Locale.ROOT, "%04X", 0x30 + UCharacter.getNumericValue(cp)));
+ builder.append(
+ String.format(Locale.ROOT, "%04X", 0x30 + UCharacter.getNumericValue(cp)));
builder.append(" # ").append(UCharacter.getName(cp));
builder.append("\n");
}
diff --git a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/RBBIRuleCompiler.java b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/RBBIRuleCompiler.java
index 50e1275..a210244 100644
--- a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/RBBIRuleCompiler.java
+++ b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/RBBIRuleCompiler.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene.analysis.icu;
-
+import com.ibm.icu.text.RuleBasedBreakIterator;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -27,42 +27,42 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
-import com.ibm.icu.text.RuleBasedBreakIterator;
-
/**
- * Command-line utility to converts RuleBasedBreakIterator (.rbbi) files into
- * binary compiled form (.brk).
+ * Command-line utility to converts RuleBasedBreakIterator (.rbbi) files into binary compiled form
+ * (.brk).
*/
public class RBBIRuleCompiler {
-
+
static String getRules(File ruleFile) throws IOException {
StringBuilder rules = new StringBuilder();
InputStream in = new FileInputStream(ruleFile);
BufferedReader cin = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
String line = null;
while ((line = cin.readLine()) != null) {
- if (!line.startsWith("#"))
+ if (!line.startsWith("#")) {
rules.append(line);
+ }
rules.append('\n');
}
cin.close();
in.close();
return rules.toString();
}
-
+
static void compile(File srcDir, File destDir) throws Exception {
- File files[] = srcDir.listFiles(new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.endsWith("rbbi");
- }});
+ File files[] =
+ srcDir.listFiles(
+ new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.endsWith("rbbi");
+ }
+ });
if (files == null) throw new IOException("Path does not exist: " + srcDir);
for (int i = 0; i < files.length; i++) {
File file = files[i];
- File outputFile = new File(destDir,
- file.getName().replaceAll("rbbi$", "brk"));
+ File outputFile = new File(destDir, file.getName().replaceAll("rbbi$", "brk"));
String rules = getRules(file);
- System.err.print("Compiling " + file.getName() + " to "
- + outputFile.getName() + ": ");
+ System.err.print("Compiling " + file.getName() + " to " + outputFile.getName() + ": ");
/*
* if there is a syntax error, compileRules() may succeed. the way to
* check is to try to instantiate from the string. additionally if the
@@ -84,7 +84,7 @@ public class RBBIRuleCompiler {
System.err.println(outputFile.length() + " bytes.");
}
}
-
+
public static void main(String args[]) throws Exception {
if (args.length < 2) {
System.err.println("Usage: RBBIRuleComputer <sourcedir> <destdir>");
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
index df3c067..de3748b 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
@@ -16,36 +16,31 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.util.HashMap;
import java.util.Map;
-
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
import org.apache.lucene.analysis.ja.dict.Dictionary;
-
// TODO: would be nice to show 2nd best path in a diff't
// color...
-/**
- * Outputs the dot (graphviz) string for the viterbi lattice.
- */
+/** Outputs the dot (graphviz) string for the viterbi lattice. */
public class GraphvizFormatter {
-
- private final static String BOS_LABEL = "BOS";
-
- private final static String EOS_LABEL = "EOS";
-
- private final static String FONT_NAME = "Helvetica";
-
+
+ private static final String BOS_LABEL = "BOS";
+
+ private static final String EOS_LABEL = "EOS";
+
+ private static final String FONT_NAME = "Helvetica";
+
private final ConnectionCosts costs;
-
+
private final Map<String, String> bestPathMap;
-
+
private final StringBuilder sb = new StringBuilder();
-
+
public GraphvizFormatter(ConnectionCosts costs) {
this.costs = costs;
this.bestPathMap = new HashMap<>();
@@ -60,7 +55,14 @@ public class GraphvizFormatter {
}
// Backtraces another incremental fragment:
- void onBacktrace(JapaneseTokenizer tok, WrappedPositionArray positions, int lastBackTracePos, Position endPosData, int fromIDX, char[] fragment, boolean isEnd) {
+ void onBacktrace(
+ JapaneseTokenizer tok,
+ WrappedPositionArray positions,
+ int lastBackTracePos,
+ Position endPosData,
+ int fromIDX,
+ char[] fragment,
+ boolean isEnd) {
setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);
sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));
if (isEnd) {
@@ -72,7 +74,8 @@ public class GraphvizFormatter {
}
// Records which arcs make up the best bath:
- private void setBestPathMap(WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {
+ private void setBestPathMap(
+ WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {
bestPathMap.clear();
int pos = endPosData.pos;
@@ -85,7 +88,7 @@ public class GraphvizFormatter {
final String toNodeID = getNodeID(pos, bestIDX);
final String fromNodeID = getNodeID(backPos, backIDX);
-
+
assert !bestPathMap.containsKey(fromNodeID);
assert !bestPathMap.containsValue(toNodeID);
bestPathMap.put(fromNodeID, toNodeID);
@@ -93,14 +96,19 @@ public class GraphvizFormatter {
bestIDX = backIDX;
}
}
-
- private String formatNodes(JapaneseTokenizer tok, WrappedPositionArray positions, int startPos, Position endPosData, char[] fragment) {
+
+ private String formatNodes(
+ JapaneseTokenizer tok,
+ WrappedPositionArray positions,
+ int startPos,
+ Position endPosData,
+ char[] fragment) {
StringBuilder sb = new StringBuilder();
// Output nodes
- for (int pos = startPos+1; pos <= endPosData.pos; pos++) {
+ for (int pos = startPos + 1; pos <= endPosData.pos; pos++) {
final Position posData = positions.get(pos);
- for(int idx=0;idx<posData.count;idx++) {
+ for (int idx = 0; idx < posData.count; idx++) {
sb.append(" ");
sb.append(getNodeID(pos, idx));
sb.append(" [label=\"");
@@ -114,7 +122,7 @@ public class GraphvizFormatter {
// Output arcs
for (int pos = endPosData.pos; pos > startPos; pos--) {
final Position posData = positions.get(pos);
- for(int idx=0;idx<posData.count;idx++) {
+ for (int idx = 0; idx < posData.count; idx++) {
final Position backPosData = positions.get(posData.backPos[idx]);
final String toNodeID = getNodeID(pos, idx);
final String fromNodeID = getNodeID(posData.backPos[idx], posData.backIndex[idx]);
@@ -134,13 +142,14 @@ public class GraphvizFormatter {
final Dictionary dict = tok.getDict(posData.backType[idx]);
final int wordCost = dict.getWordCost(posData.backID[idx]);
- final int bgCost = costs.get(backPosData.lastRightID[posData.backIndex[idx]],
- dict.getLeftId(posData.backID[idx]));
+ final int bgCost =
+ costs.get(
+ backPosData.lastRightID[posData.backIndex[idx]],
+ dict.getLeftId(posData.backID[idx]));
+
+ final String surfaceForm =
+ new String(fragment, posData.backPos[idx] - startPos, pos - posData.backPos[idx]);
- final String surfaceForm = new String(fragment,
- posData.backPos[idx] - startPos,
- pos - posData.backPos[idx]);
-
sb.append(" [label=\"");
sb.append(surfaceForm);
sb.append(' ');
@@ -156,25 +165,29 @@ public class GraphvizFormatter {
}
return sb.toString();
}
-
+
private String formatHeader() {
StringBuilder sb = new StringBuilder();
sb.append("digraph viterbi {\n");
- sb.append(" graph [ fontsize=30 labelloc=\"t\" label=\"\" splines=true overlap=false rankdir = \"LR\"];\n");
- //sb.append(" // A2 paper size\n");
- //sb.append(" size = \"34.4,16.5\";\n");
- //sb.append(" // try to fill paper\n");
- //sb.append(" ratio = fill;\n");
+ sb.append(
+ " graph [ fontsize=30 labelloc=\"t\" label=\"\" splines=true overlap=false rankdir = \"LR\"];\n");
+ // sb.append(" // A2 paper size\n");
+ // sb.append(" size = \"34.4,16.5\";\n");
+ // sb.append(" // try to fill paper\n");
+ // sb.append(" ratio = fill;\n");
sb.append(" edge [ fontname=\"" + FONT_NAME + "\" fontcolor=\"red\" color=\"#606060\" ]\n");
- sb.append(" node [ style=\"filled\" fillcolor=\"#e8e8f0\" shape=\"Mrecord\" fontname=\"" + FONT_NAME + "\" ]\n");
-
+ sb.append(
+ " node [ style=\"filled\" fillcolor=\"#e8e8f0\" shape=\"Mrecord\" fontname=\""
+ + FONT_NAME
+ + "\" ]\n");
+
return sb.toString();
}
-
+
private String formatTrailer() {
return "}";
}
-
+
private String getNodeID(int pos, int idx) {
return pos + "." + idx;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
index e2f5959..4752dc1 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
@@ -16,11 +16,9 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
@@ -33,37 +31,42 @@ import org.apache.lucene.analysis.ja.dict.UserDictionary;
/**
* Analyzer for Japanese that uses morphological analysis.
- * @see JapaneseTokenizer
*
+ * @see JapaneseTokenizer
* @since 3.6.0
*/
public class JapaneseAnalyzer extends StopwordAnalyzerBase {
private final Mode mode;
private final Set<String> stoptags;
private final UserDictionary userDict;
-
+
public JapaneseAnalyzer() {
- this(null, JapaneseTokenizer.DEFAULT_MODE, DefaultSetHolder.DEFAULT_STOP_SET, DefaultSetHolder.DEFAULT_STOP_TAGS);
+ this(
+ null,
+ JapaneseTokenizer.DEFAULT_MODE,
+ DefaultSetHolder.DEFAULT_STOP_SET,
+ DefaultSetHolder.DEFAULT_STOP_TAGS);
}
-
- public JapaneseAnalyzer(UserDictionary userDict, Mode mode, CharArraySet stopwords, Set<String> stoptags) {
+
+ public JapaneseAnalyzer(
+ UserDictionary userDict, Mode mode, CharArraySet stopwords, Set<String> stoptags) {
super(stopwords);
this.userDict = userDict;
this.mode = mode;
this.stoptags = stoptags;
}
-
- public static CharArraySet getDefaultStopSet(){
+
+ public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
- public static Set<String> getDefaultStopTags(){
+
+ public static Set<String> getDefaultStopTags() {
return DefaultSetHolder.DEFAULT_STOP_TAGS;
}
-
+
/**
- * Atomically loads DEFAULT_STOP_SET, DEFAULT_STOP_TAGS in a lazy fashion once the
- * outer class accesses the static final set the first time.
+ * Atomically loads DEFAULT_STOP_SET, DEFAULT_STOP_TAGS in a lazy fashion once the outer class
+ * accesses the static final set the first time.
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
@@ -71,8 +74,10 @@ public class JapaneseAnalyzer extends StopwordAnalyzerBase {
static {
try {
- DEFAULT_STOP_SET = loadStopwordSet(true, JapaneseAnalyzer.class, "stopwords.txt", "#"); // ignore case
- final CharArraySet tagset = loadStopwordSet(false, JapaneseAnalyzer.class, "stoptags.txt", "#");
+ DEFAULT_STOP_SET =
+ loadStopwordSet(true, JapaneseAnalyzer.class, "stopwords.txt", "#"); // ignore case
+ final CharArraySet tagset =
+ loadStopwordSet(false, JapaneseAnalyzer.class, "stoptags.txt", "#");
DEFAULT_STOP_TAGS = new HashSet<>();
for (Object element : tagset) {
char chars[] = (char[]) element;
@@ -84,7 +89,7 @@ public class JapaneseAnalyzer extends StopwordAnalyzerBase {
}
}
}
-
+
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new JapaneseTokenizer(userDict, true, true, mode);
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java
index 255ea2c..6863ecb 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ja.tokenattributes.BaseFormAttribute;
@@ -28,13 +26,12 @@ import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
/**
* Replaces term text with the {@link BaseFormAttribute}.
- * <p>
- * This acts as a lemmatizer for verbs and adjectives.
- * <p>
- * To prevent terms from being stemmed use an instance of
- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
- * the {@link KeywordAttribute} before this {@link TokenStream}.
- * </p>
+ *
+ * <p>This acts as a lemmatizer for verbs and adjectives.
+ *
+ * <p>To prevent terms from being stemmed use an instance of {@link SetKeywordMarkerFilter} or a
+ * custom {@link TokenFilter} that sets the {@link KeywordAttribute} before this {@link
+ * TokenStream}.
*/
public final class JapaneseBaseFormFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilterFactory.java
index 8ad2771..9a63cad 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapaneseBaseFormFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -32,6 +31,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* </analyzer>
* </fieldType>
* </pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
@@ -41,13 +41,13 @@ public class JapaneseBaseFormFilterFactory extends TokenFilterFactory {
public static final String NAME = "japaneseBaseForm";
/** Creates a new JapaneseBaseFormFilterFactory */
- public JapaneseBaseFormFilterFactory(Map<String,String> args) {
+ public JapaneseBaseFormFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public JapaneseBaseFormFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
index 7e1d7a1..c2350c7 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java
@@ -16,52 +16,45 @@
*/
package org.apache.lucene.analysis.ja;
-
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.analysis.util.RollingCharBuffer;
-
import java.io.IOException;
import java.io.Reader;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.analysis.util.RollingCharBuffer;
/**
* Normalizes Japanese horizontal iteration marks (odoriji) to their expanded form.
- * <p>
- * Sequences of iteration marks are supported. In case an illegal sequence of iteration
- * marks is encountered, the implementation emits the illegal source character as-is
- * without considering its script. For example, with input "?ゝ", we get
- * "??" even though the question mark isn't hiragana.
- * </p>
- * <p>
- * Note that a full stop punctuation character "。" (U+3002) can not be iterated
- * (see below). Iteration marks themselves can be emitted in case they are illegal,
- * i.e. if they go back past the beginning of the character stream.
- * </p>
- * <p>
- * The implementation buffers input until a full stop punctuation character (U+3002)
- * or EOF is reached in order to not keep a copy of the character stream in memory.
- * Vertical iteration marks, which are even rarer than horizontal iteration marks in
- * contemporary Japanese, are unsupported.
- * </p>
+ *
+ * <p>Sequences of iteration marks are supported. In case an illegal sequence of iteration marks is
+ * encountered, the implementation emits the illegal source character as-is without considering its
+ * script. For example, with input "?ゝ", we get "??" even though the question mark isn't hiragana.
+ *
+ * <p>Note that a full stop punctuation character "。" (U+3002) can not be iterated (see below).
+ * Iteration marks themselves can be emitted in case they are illegal, i.e. if they go back past the
+ * beginning of the character stream.
+ *
+ * <p>The implementation buffers input until a full stop punctuation character (U+3002) or EOF is
+ * reached in order to not keep a copy of the character stream in memory. Vertical iteration marks,
+ * which are even rarer than horizontal iteration marks in contemporary Japanese, are unsupported.
*/
public class JapaneseIterationMarkCharFilter extends CharFilter {
/** Normalize kanji iteration marks by default */
- public static final boolean NORMALIZE_KANJI_DEFAULT = true;
+ public static final boolean NORMALIZE_KANJI_DEFAULT = true;
/** Normalize kana iteration marks by default */
public static final boolean NORMALIZE_KANA_DEFAULT = true;
- private static final char KANJI_ITERATION_MARK = '\u3005'; // 々
+ private static final char KANJI_ITERATION_MARK = '\u3005'; // 々
- private static final char HIRAGANA_ITERATION_MARK = '\u309d'; // ゝ
+ private static final char HIRAGANA_ITERATION_MARK = '\u309d'; // ゝ
private static final char HIRAGANA_VOICED_ITERATION_MARK = '\u309e'; // ゞ
- private static final char KATAKANA_ITERATION_MARK = '\u30fd'; // ヽ
+ private static final char KATAKANA_ITERATION_MARK = '\u30fd'; // ヽ
private static final char KATAKANA_VOICED_ITERATION_MARK = '\u30fe'; // ヾ
- private static final char FULL_STOP_PUNCTUATION = '\u3002'; // 。
+ private static final char FULL_STOP_PUNCTUATION = '\u3002'; // 。
// Hiragana to dakuten map (lookup using code point - 0x30ab（か）*/
private static char[] h2d = new char[50];
@@ -83,16 +76,16 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
static {
// Hiragana dakuten map
- h2d[0] = '\u304c'; // か => が
- h2d[1] = '\u304c'; // が => が
- h2d[2] = '\u304e'; // き => ぎ
- h2d[3] = '\u304e'; // ぎ => ぎ
- h2d[4] = '\u3050'; // く => ぐ
- h2d[5] = '\u3050'; // ぐ => ぐ
- h2d[6] = '\u3052'; // け => げ
- h2d[7] = '\u3052'; // げ => げ
- h2d[8] = '\u3054'; // こ => ご
- h2d[9] = '\u3054'; // ご => ご
+ h2d[0] = '\u304c'; // か => が
+ h2d[1] = '\u304c'; // が => が
+ h2d[2] = '\u304e'; // き => ぎ
+ h2d[3] = '\u304e'; // ぎ => ぎ
+ h2d[4] = '\u3050'; // く => ぐ
+ h2d[5] = '\u3050'; // ぐ => ぐ
+ h2d[6] = '\u3052'; // け => げ
+ h2d[7] = '\u3052'; // げ => げ
+ h2d[8] = '\u3054'; // こ => ご
+ h2d[9] = '\u3054'; // ご => ご
h2d[10] = '\u3056'; // さ => ざ
h2d[11] = '\u3056'; // ざ => ざ
h2d[12] = '\u3058'; // し => じ
@@ -151,15 +144,15 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
this(input, NORMALIZE_KANJI_DEFAULT, NORMALIZE_KANA_DEFAULT);
}
-
/**
* Constructor
*
- * @param input char stream
+ * @param input char stream
* @param normalizeKanji indicates whether kanji iteration marks should be normalized
* @param normalizeKana indicates whether kana iteration marks should be normalized
*/
- public JapaneseIterationMarkCharFilter(Reader input, boolean normalizeKanji, boolean normalizeKana) {
+ public JapaneseIterationMarkCharFilter(
+ Reader input, boolean normalizeKanji, boolean normalizeKana) {
super(input);
this.normalizeKanji = normalizeKanji;
this.normalizeKana = normalizeKana;
@@ -191,7 +184,7 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
buffer.freeBefore(bufferPosition);
return ic;
}
-
+
char c = (char) ic;
// Skip surrogate pair characters
@@ -204,12 +197,12 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
buffer.freeBefore(bufferPosition);
iterationMarkSpanEndPosition = bufferPosition + 1;
}
-
+
// Normalize iteration mark
if (isIterationMark(c)) {
c = normalizeIterationMark(c);
}
-
+
bufferPosition++;
return c;
}
@@ -250,7 +243,9 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
*/
private int nextIterationMarkSpanSize() throws IOException {
int spanSize = 0;
- for (int i = bufferPosition; buffer.get(i) != -1 && isIterationMark((char) (buffer.get(i))); i++) {
+ for (int i = bufferPosition;
+ buffer.get(i) != -1 && isIterationMark((char) (buffer.get(i)));
+ i++) {
spanSize++;
}
// Restrict span size so that we don't go past the previous end position
@@ -288,7 +283,8 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
return normalizedKatakana(c, m);
}
- return c; // If m is not kana and we are to normalize it, we assume it is kanji and simply return it
+ return c; // If m is not kana and we are to normalize it, we assume it is kanji and simply
+ // return it
}
/**
@@ -331,7 +327,7 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
* Iteration mark character predicate
*
* @param c character to test
- * @return true if c is an iteration mark character. Otherwise false.
+ * @return true if c is an iteration mark character. Otherwise false.
*/
private boolean isIterationMark(char c) {
return isKanjiIterationMark(c) || isHiraganaIterationMark(c) || isKatakanaIterationMark(c);
@@ -341,7 +337,7 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
* Hiragana iteration mark character predicate
*
* @param c character to test
- * @return true if c is a hiragana iteration mark character. Otherwise false.
+ * @return true if c is a hiragana iteration mark character. Otherwise false.
*/
private boolean isHiraganaIterationMark(char c) {
if (normalizeKana) {
@@ -355,7 +351,7 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
* Katakana iteration mark character predicate
*
* @param c character to test
- * @return true if c is a katakana iteration mark character. Otherwise false.
+ * @return true if c is a katakana iteration mark character. Otherwise false.
*/
private boolean isKatakanaIterationMark(char c) {
if (normalizeKana) {
@@ -369,7 +365,7 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
* Kanji iteration mark character predicate
*
* @param c character to test
- * @return true if c is a kanji iteration mark character. Otherwise false.
+ * @return true if c is a kanji iteration mark character. Otherwise false.
*/
private boolean isKanjiIterationMark(char c) {
if (normalizeKanji) {
@@ -420,11 +416,11 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
}
/**
- * Looks up a character in dakuten map and returns the dakuten variant if it exists.
- * Otherwise return the character being looked up itself
+ * Looks up a character in dakuten map and returns the dakuten variant if it exists. Otherwise
+ * return the character being looked up itself
*
- * @param c character to look up
- * @param map dakuten map
+ * @param c character to look up
+ * @param map dakuten map
* @param offset code point offset from c
* @return mapped character or c if no mapping exists
*/
@@ -439,8 +435,8 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
/**
* Predicate indicating if the lookup character is within dakuten map range
*
- * @param c character to look up
- * @param map dakuten map
+ * @param c character to look up
+ * @param map dakuten map
* @param offset code point offset from c
* @return true if c is mapped by map and otherwise false
*/
@@ -448,7 +444,6 @@ public class JapaneseIterationMarkCharFilter extends CharFilter {
return c >= offset && c < offset + map.length;
}
-
@Override
protected int correct(int currentOff) {
return currentOff; // this filter doesn't change the length of strings
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
index 2063be7..c300cc0 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.Reader;
import java.util.Map;
-
import org.apache.lucene.analysis.CharFilterFactory;
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapaneseIterationMarkCharFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
* <analyzer>
@@ -45,12 +44,16 @@ public class JapaneseIterationMarkCharFilterFactory extends CharFilterFactory {
private final boolean normalizeKanji;
private final boolean normalizeKana;
-
+
/** Creates a new JapaneseIterationMarkCharFilterFactory */
- public JapaneseIterationMarkCharFilterFactory(Map<String,String> args) {
+ public JapaneseIterationMarkCharFilterFactory(Map<String, String> args) {
super(args);
- normalizeKanji = getBoolean(args, NORMALIZE_KANJI_PARAM, JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT);
- normalizeKana = getBoolean(args, NORMALIZE_KANA_PARAM, JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT);
+ normalizeKanji =
+ getBoolean(
+ args, NORMALIZE_KANJI_PARAM, JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT);
+ normalizeKana =
+ getBoolean(
+ args, NORMALIZE_KANA_PARAM, JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java
index e97b2c6..18b5ee1 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java
@@ -16,34 +16,28 @@
*/
package org.apache.lucene.analysis.ja;
-
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import java.io.IOException;
-
/**
- * A {@link TokenFilter} that normalizes common katakana spelling variations
- * ending in a long sound character by removing this character (U+30FC). Only
- * katakana words longer than a minimum length are stemmed (default is four).
- * <p>
- * Note that only full-width katakana characters are supported. Please use a
- * {@link org.apache.lucene.analysis.cjk.CJKWidthFilter} to convert half-width
- * katakana to full-width before using this filter.
- * </p>
- * <p>
- * In order to prevent terms from being stemmed, use an instance of
- * {@link org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter}
- * or a custom {@link TokenFilter} that sets the {@link KeywordAttribute}
- * before this {@link TokenStream}.
- * </p>
+ * A {@link TokenFilter} that normalizes common katakana spelling variations ending in a long sound
+ * character by removing this character (U+30FC). Only katakana words longer than a minimum length
+ * are stemmed (default is four).
+ *
+ * <p>Note that only full-width katakana characters are supported. Please use a {@link
+ * org.apache.lucene.analysis.cjk.CJKWidthFilter} to convert half-width katakana to full-width
+ * before using this filter.
+ *
+ * <p>In order to prevent terms from being stemmed, use an instance of {@link
+ * org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter} or a custom {@link TokenFilter}
+ * that sets the {@link KeywordAttribute} before this {@link TokenStream}.
*/
-
public final class JapaneseKatakanaStemFilter extends TokenFilter {
- public final static int DEFAULT_MINIMUM_LENGTH = 4;
- private final static char HIRAGANA_KATAKANA_PROLONGED_SOUND_MARK = '\u30fc';
+ public static final int DEFAULT_MINIMUM_LENGTH = 4;
+ private static final char HIRAGANA_KATAKANA_PROLONGED_SOUND_MARK = '\u30fc';
private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
@@ -75,7 +69,7 @@ public final class JapaneseKatakanaStemFilter extends TokenFilter {
return length;
}
- if (! isKatakana(term, length)) {
+ if (!isKatakana(term, length)) {
return length;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java
index 2f75552..5ec8922 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ja;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.TokenFilterFactory;
-
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link JapaneseKatakanaStemFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -33,6 +32,7 @@ import java.util.Map;
* </analyzer>
* </fieldType>
* </pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
@@ -43,13 +43,15 @@ public class JapaneseKatakanaStemFilterFactory extends TokenFilterFactory {
private static final String MINIMUM_LENGTH_PARAM = "minimumLength";
private final int minimumLength;
-
+
/** Creates a new JapaneseKatakanaStemFilterFactory */
- public JapaneseKatakanaStemFilterFactory(Map<String,String> args) {
+ public JapaneseKatakanaStemFilterFactory(Map<String, String> args) {
super(args);
- minimumLength = getInt(args, MINIMUM_LENGTH_PARAM, JapaneseKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH);
+ minimumLength =
+ getInt(args, MINIMUM_LENGTH_PARAM, JapaneseKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH);
if (minimumLength < 2) {
- throw new IllegalArgumentException("Illegal " + MINIMUM_LENGTH_PARAM + " " + minimumLength + " (must be 2 or greater)");
+ throw new IllegalArgumentException(
+ "Illegal " + MINIMUM_LENGTH_PARAM + " " + minimumLength + " (must be 2 or greater)");
}
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java
index 9f4c1d5..4fe2062 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.IOException;
import java.math.BigDecimal;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -29,68 +27,68 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
/**
- * A {@link TokenFilter} that normalizes Japanese numbers (kansūji) to regular Arabic
- * decimal numbers in half-width characters.
- * <p>
- * Japanese numbers are often written using a combination of kanji and Arabic numbers with
- * various kinds punctuation. For example, ３．２千 means 3200. This filter does this kind
- * of normalization and allows a search for 3200 to match ３．２千 in text, but can also be
- * used to make range facets based on the normalized numbers and so on.
- * <p>
- * Notice that this analyzer uses a token composition scheme and relies on punctuation
- * tokens being found in the token stream. Please make sure your {@link JapaneseTokenizer}
- * has {@code discardPunctuation} set to false. In case punctuation characters, such as ．
- * (U+FF0E FULLWIDTH FULL STOP), is removed from the token stream, this filter would find
- * input tokens tokens ３ and ２千 and give outputs 3 and 2000 instead of 3200, which is
- * likely not the intended result. If you want to remove punctuation characters from your
- * index that are not part of normalized numbers, add a
- * {@link org.apache.lucene.analysis.StopFilter} with the punctuation you wish to
- * remove after {@link JapaneseNumberFilter} in your analyzer chain.
- * <p>
- * Below are some examples of normalizations this filter supports. The input is untokenized
- * text and the result is the single term attribute emitted for the input.
+ * A {@link TokenFilter} that normalizes Japanese numbers (kansūji) to regular Arabic decimal
+ * numbers in half-width characters.
+ *
+ * <p>Japanese numbers are often written using a combination of kanji and Arabic numbers with
+ * various kinds punctuation. For example, ３．２千 means 3200. This filter does this kind of
+ * normalization and allows a search for 3200 to match ３．２千 in text, but can also be used to make
+ * range facets based on the normalized numbers and so on.
+ *
+ * <p>Notice that this analyzer uses a token composition scheme and relies on punctuation tokens
+ * being found in the token stream. Please make sure your {@link JapaneseTokenizer} has {@code
+ * discardPunctuation} set to false. In case punctuation characters, such as ． (U+FF0E FULLWIDTH
+ * FULL STOP), is removed from the token stream, this filter would find input tokens tokens ３ and ２千
+ * and give outputs 3 and 2000 instead of 3200, which is likely not the intended result. If you want
+ * to remove punctuation characters from your index that are not part of normalized numbers, add a
+ * {@link org.apache.lucene.analysis.StopFilter} with the punctuation you wish to remove after
+ * {@link JapaneseNumberFilter} in your analyzer chain.
+ *
+ * <p>Below are some examples of normalizations this filter supports. The input is untokenized text
+ * and the result is the single term attribute emitted for the input.
+ *
* <ul>
- * <li>〇〇七 becomes 7</li>
- * <li>一〇〇〇 becomes 1000</li>
- * <li>三千2百２十三 becomes 3223</li>
- * <li>兆六百万五千一 becomes 1000006005001</li>
- * <li>３．２千 becomes 3200</li>
- * <li>１．２万３４５．６７ becomes 12345.67</li>
- * <li>4,647.100 becomes 4647.1</li>
- * <li>15,7 becomes 157 (be aware of this weakness)</li>
+ * <li>〇〇七 becomes 7
+ * <li>一〇〇〇 becomes 1000
+ * <li>三千2百２十三 becomes 3223
+ * <li>兆六百万五千一 becomes 1000006005001
+ * <li>３．２千 becomes 3200
+ * <li>１．２万３４５．６７ becomes 12345.67
+ * <li>4,647.100 becomes 4647.1
+ * <li>15,7 becomes 157 (be aware of this weakness)
* </ul>
- * <p>
- * Tokens preceded by a token with {@link PositionIncrementAttribute} of zero are left
- * left untouched and emitted as-is.
- * <p>
- * This filter does not use any part-of-speech information for its normalization and
- * the motivation for this is to also support n-grammed token streams in the future.
- * <p>
- * This filter may in some cases normalize tokens that are not numbers in their context.
- * For example, is 田中京一 is a name and means Tanaka Kyōichi, but 京一 (Kyōichi) out of
- * context can strictly speaking also represent the number 10000000000000001. This filter
- * respects the {@link KeywordAttribute}, which can be used to prevent specific
- * normalizations from happening.
- * <p>
- * Also notice that token attributes such as
- * {@link org.apache.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute},
- * {@link org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute},
- * {@link org.apache.lucene.analysis.ja.tokenattributes.InflectionAttribute} and
- * {@link org.apache.lucene.analysis.ja.tokenattributes.BaseFormAttribute} are left
- * unchanged and will inherit the values of the last token used to compose the normalized
- * number and can be wrong. Hence, for １０万 (10000), we will have
- * {@link org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute}
- * set to マン. This is a known issue and is subject to a future improvement.
- * <p>
- * Japanese formal numbers (daiji), accounting numbers and decimal fractions are currently
- * not supported.
+ *
+ * <p>Tokens preceded by a token with {@link PositionIncrementAttribute} of zero are left left
+ * untouched and emitted as-is.
+ *
+ * <p>This filter does not use any part-of-speech information for its normalization and the
+ * motivation for this is to also support n-grammed token streams in the future.
+ *
+ * <p>This filter may in some cases normalize tokens that are not numbers in their context. For
+ * example, is 田中京一 is a name and means Tanaka Kyōichi, but 京一 (Kyōichi) out of context can strictly
+ * speaking also represent the number 10000000000000001. This filter respects the {@link
+ * KeywordAttribute}, which can be used to prevent specific normalizations from happening.
+ *
+ * <p>Also notice that token attributes such as {@link
+ * org.apache.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute}, {@link
+ * org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute}, {@link
+ * org.apache.lucene.analysis.ja.tokenattributes.InflectionAttribute} and {@link
+ * org.apache.lucene.analysis.ja.tokenattributes.BaseFormAttribute} are left unchanged and will
+ * inherit the values of the last token used to compose the normalized number and can be wrong.
+ * Hence, for １０万 (10000), we will have {@link
+ * org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute} set to マン. This is a known issue
+ * and is subject to a future improvement.
+ *
+ * <p>Japanese formal numbers (daiji), accounting numbers and decimal fractions are currently not
+ * supported.
*/
public class JapaneseNumberFilter extends TokenFilter {
private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttr = addAttribute(OffsetAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
- private final PositionIncrementAttribute posIncrAttr = addAttribute(PositionIncrementAttribute.class);
+ private final PositionIncrementAttribute posIncrAttr =
+ addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLengthAttr = addAttribute(PositionLengthAttribute.class);
private static char NO_NUMERAL = Character.MAX_VALUE;
@@ -104,7 +102,7 @@ public class JapaneseNumberFilter extends TokenFilter {
private StringBuilder numeral;
private int fallThroughTokens;
-
+
private boolean exhausted = false;
static {
@@ -127,11 +125,11 @@ public class JapaneseNumberFilter extends TokenFilter {
for (int i = 0; i < exponents.length; i++) {
exponents[i] = 0;
}
- exponents['十'] = 1; // 十 U+5341 10
- exponents['百'] = 2; // 百 U+767E 100
- exponents['千'] = 3; // 千 U+5343 1,000
- exponents['万'] = 4; // 万 U+4E07 10,000
- exponents['億'] = 8; // 億 U+5104 100,000,000
+ exponents['十'] = 1; // 十 U+5341 10
+ exponents['百'] = 2; // 百 U+767E 100
+ exponents['千'] = 3; // 千 U+5343 1,000
+ exponents['万'] = 4; // 万 U+4E07 10,000
+ exponents['億'] = 8; // 億 U+5104 100,000,000
exponents['兆'] = 12; // 兆 U+5146 1,000,000,000,000
exponents['京'] = 16; // 京 U+4EAC 10,000,000,000,000,000
exponents['垓'] = 20; // 垓 U+5793 100,000,000,000,000,000,000
@@ -154,7 +152,7 @@ public class JapaneseNumberFilter extends TokenFilter {
if (exhausted) {
return false;
}
-
+
if (!input.incrementToken()) {
exhausted = true;
return false;
@@ -181,7 +179,7 @@ public class JapaneseNumberFilter extends TokenFilter {
State preCompositionState = captureState();
String term = termAttr.toString();
boolean numeralTerm = isNumeral(term);
-
+
while (moreTokens && numeralTerm) {
if (!composedNumberToken) {
@@ -311,8 +309,8 @@ public class JapaneseNumberFilter extends TokenFilter {
}
/**
- * Parses a "medium sized" number, typically less than 10,000（万）, but might be larger
- * due to a larger factor from {link parseBasicNumber}.
+ * Parses a "medium sized" number, typically less than 10,000（万）, but might be larger due to a
+ * larger factor from {link parseBasicNumber}.
*
* @param buffer buffer to parse
* @return parsed number, or null on error or end of input
@@ -365,7 +363,8 @@ public class JapaneseNumberFilter extends TokenFilter {
}
/**
- * Parse a basic number, which is a sequence of Arabic numbers or a sequence or 0-9 kanji numerals (〇 to 九).
+ * Parse a basic number, which is a sequence of Arabic numbers or a sequence or 0-9 kanji numerals
+ * (〇 to 九).
*
* @param buffer buffer to parse
* @return parsed number, or null on error or end of input
@@ -535,8 +534,8 @@ public class JapaneseNumberFilter extends TokenFilter {
}
/**
- * Returns the numeric value for the specified character Arabic numeral.
- * Behavior is undefined if a non-Arabic numeral is provided
+ * Returns the numeric value for the specified character Arabic numeral. Behavior is undefined if
+ * a non-Arabic numeral is provided
*
* @param c arabic numeral character
* @return numeral value
@@ -552,8 +551,8 @@ public class JapaneseNumberFilter extends TokenFilter {
}
/**
- * Kanji numeral predicate that tests if the provided character is one of 〇, 一, 二, 三, 四, 五, 六, 七, 八, or 九.
- * Larger number kanji gives a false value.
+ * Kanji numeral predicate that tests if the provided character is one of 〇, 一, 二, 三, 四, 五, 六, 七,
+ * 八, or 九. Larger number kanji gives a false value.
*
* @param c character to test
* @return true if and only is character is one of 〇, 一, 二, 三, 四, 五, 六, 七, 八, or 九 (0 to 9)
@@ -581,7 +580,7 @@ public class JapaneseNumberFilter extends TokenFilter {
* @return true if and only if c is a decimal point
*/
private boolean isDecimalPoint(char c) {
- return c == '.' // U+002E FULL STOP
+ return c == '.' // U+002E FULL STOP
|| c == '．'; // U+FF0E FULLWIDTH FULL STOP
}
@@ -592,13 +591,11 @@ public class JapaneseNumberFilter extends TokenFilter {
* @return true if and only if c is a thousand separator predicate
*/
private boolean isThousandSeparator(char c) {
- return c == ',' // U+002C COMMA
+ return c == ',' // U+002C COMMA
|| c == '，'; // U+FF0C FULLWIDTH COMMA
}
- /**
- * Buffer that holds a Japanese number string and a position index used as a parsed-to marker
- */
+ /** Buffer that holds a Japanese number string and a position index used as a parsed-to marker */
public static class NumberBuffer {
private int position;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilterFactory.java
index 7e7b559..5b1b92e 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilterFactory.java
@@ -16,15 +16,13 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
- * Factory for {@link JapaneseNumberFilter}.
- * <br>
+ * Factory for {@link JapaneseNumberFilter}. <br>
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -33,9 +31,10 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* </analyzer>
* </fieldType>
* </pre>
- * <p>
- * It is important that punctuation is not discarded by the tokenizer so use
- * {@code discardPunctuation="false"} in your {@link JapaneseTokenizerFactory}.
+ *
+ * <p>It is important that punctuation is not discarded by the tokenizer so use {@code
+ * discardPunctuation="false"} in your {@link JapaneseTokenizerFactory}.
+ *
* @since 6.0.0
* @lucene.spi {@value #NAME}
*/
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java
index 342295d..89fe3d4 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java
@@ -16,23 +16,20 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.util.Set;
-
import org.apache.lucene.analysis.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute;
-/**
- * Removes tokens that match a set of part-of-speech tags.
- */
+/** Removes tokens that match a set of part-of-speech tags. */
public final class JapanesePartOfSpeechStopFilter extends FilteringTokenFilter {
private final Set<String> stopTags;
private final PartOfSpeechAttribute posAtt = addAttribute(PartOfSpeechAttribute.class);
/**
* Create a new {@link JapanesePartOfSpeechStopFilter}.
- * @param input the {@link TokenStream} to consume
+ *
+ * @param input the {@link TokenStream} to consume
* @param stopTags the part-of-speech tags that should be removed
*/
public JapanesePartOfSpeechStopFilter(TokenStream input, Set<String> stopTags) {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java
index 0e69d01..503ad93 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java
@@ -16,20 +16,19 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-
import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.TokenFilterFactory;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
-import org.apache.lucene.analysis.TokenFilterFactory;
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapanesePartOfSpeechStopFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -39,10 +38,12 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* </analyzer>
* </fieldType>
* </pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
-public class JapanesePartOfSpeechStopFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
+public class JapanesePartOfSpeechStopFilterFactory extends TokenFilterFactory
+ implements ResourceLoaderAware {
public static final String NAME = "japanesePartOfSpeechStop";
@@ -50,7 +51,7 @@ public class JapanesePartOfSpeechStopFilterFactory extends TokenFilterFactory im
private Set<String> stopTags;
/** Creates a new JapanesePartOfSpeechStopFilterFactory */
- public JapanesePartOfSpeechStopFilterFactory(Map<String,String> args) {
+ public JapanesePartOfSpeechStopFilterFactory(Map<String, String> args) {
super(args);
stopTagFiles = get(args, "tags");
if (stopTagFiles == null) {
@@ -60,7 +61,7 @@ public class JapanesePartOfSpeechStopFilterFactory extends TokenFilterFactory im
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public JapanesePartOfSpeechStopFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java
index 359de90..dcc8029 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java
@@ -16,21 +16,17 @@
*/
package org.apache.lucene.analysis.ja;
-
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute;
import org.apache.lucene.analysis.ja.util.ToStringUtil;
-
-import java.io.IOException;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
- * A {@link org.apache.lucene.analysis.TokenFilter} that replaces the term
- * attribute with the reading of a token in either katakana or romaji form.
- * The default reading form is katakana.
+ * A {@link org.apache.lucene.analysis.TokenFilter} that replaces the term attribute with the
+ * reading of a token in either katakana or romaji form. The default reading form is katakana.
*/
-
public final class JapaneseReadingFormFilter extends TokenFilter {
private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
private final ReadingAttribute readingAttr = addAttribute(ReadingAttribute.class);
@@ -51,7 +47,7 @@ public final class JapaneseReadingFormFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String reading = readingAttr.getReading();
-
+
if (useRomaji) {
if (reading == null) {
// if it's an OOV term, just try the term text
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilterFactory.java
index 8a3f463..f8ccc0f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ja;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.TokenFilterFactory;
-
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapaneseReadingFormFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -33,6 +32,7 @@ import java.util.Map;
* </analyzer>
* </fieldType>
* </pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
@@ -43,9 +43,9 @@ public class JapaneseReadingFormFilterFactory extends TokenFilterFactory {
private static final String ROMAJI_PARAM = "useRomaji";
private final boolean useRomaji;
-
+
/** Creates a new JapaneseReadingFormFilterFactory */
- public JapaneseReadingFormFilterFactory(Map<String,String> args) {
+ public JapaneseReadingFormFilterFactory(Map<String, String> args) {
super(args);
useRomaji = getBoolean(args, ROMAJI_PARAM, false);
if (!args.isEmpty()) {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 4a525e4..5661058 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -25,7 +25,6 @@ import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
-
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
@@ -51,72 +50,54 @@ import org.apache.lucene.util.fst.FST;
/**
* Tokenizer for Japanese that uses morphological analysis.
- * <p>
- * This tokenizer sets a number of additional attributes:
+ *
+ * <p>This tokenizer sets a number of additional attributes:
+ *
* <ul>
- * <li>{@link BaseFormAttribute} containing base form for inflected
- * adjectives and verbs.
+ * <li>{@link BaseFormAttribute} containing base form for inflected adjectives and verbs.
* <li>{@link PartOfSpeechAttribute} containing part-of-speech.
* <li>{@link ReadingAttribute} containing reading and pronunciation.
- * <li>{@link InflectionAttribute} containing additional part-of-speech
- * information for inflected forms.
+ * <li>{@link InflectionAttribute} containing additional part-of-speech information for inflected
+ * forms.
* </ul>
- * <p>
- * This tokenizer uses a rolling Viterbi search to find the
- * least cost segmentation (path) of the incoming characters.
- * For tokens that appear to be compound (> length 2 for all
- * Kanji, or > length 7 for non-Kanji), we see if there is a
- * 2nd best segmentation of that token after applying
- * penalties to the long tokens. If so, and the Mode is
- * {@link Mode#SEARCH}, we output the alternate segmentation
- * as well.
+ *
+ * <p>This tokenizer uses a rolling Viterbi search to find the least cost segmentation (path) of the
+ * incoming characters. For tokens that appear to be compound (> length 2 for all Kanji, or >
+ * length 7 for non-Kanji), we see if there is a 2nd best segmentation of that token after applying
+ * penalties to the long tokens. If so, and the Mode is {@link Mode#SEARCH}, we output the alternate
+ * segmentation as well.
*/
public final class JapaneseTokenizer extends Tokenizer {
- /**
- * Tokenization mode: this determines how the tokenizer handles
- * compound and unknown words.
- */
+ /** Tokenization mode: this determines how the tokenizer handles compound and unknown words. */
public static enum Mode {
- /**
- * Ordinary segmentation: no decomposition for compounds,
- */
+ /** Ordinary segmentation: no decomposition for compounds, */
NORMAL,
/**
- * Segmentation geared towards search: this includes a
- * decompounding process for long nouns, also including
- * the full compound token as a synonym.
+ * Segmentation geared towards search: this includes a decompounding process for long nouns,
+ * also including the full compound token as a synonym.
*/
SEARCH,
/**
* Extended mode outputs unigrams for unknown words.
+ *
* @lucene.experimental
*/
EXTENDED
}
- /**
- * Default tokenization mode. Currently this is {@link Mode#SEARCH}.
- */
+ /** Default tokenization mode. Currently this is {@link Mode#SEARCH}. */
public static final Mode DEFAULT_MODE = Mode.SEARCH;
- /**
- * Token type reflecting the original source of this token
- */
+ /** Token type reflecting the original source of this token */
public enum Type {
- /**
- * Known words from the system dictionary.
- */
+ /** Known words from the system dictionary. */
KNOWN,
- /**
- * Unknown words (heuristically segmented).
- */
+ /** Unknown words (heuristically segmented). */
UNKNOWN,
- /**
- * Known words from the user dictionary.
- */
+ /** Known words from the user dictionary. */
USER
}
@@ -181,7 +162,8 @@ public final class JapaneseTokenizer extends Tokenizer {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionIncrementAttribute posIncAtt =
+ addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLengthAtt = addAttribute(PositionLengthAttribute.class);
private final BaseFormAttribute basicFormAtt = addAttribute(BaseFormAttribute.class);
private final PartOfSpeechAttribute posAtt = addAttribute(PartOfSpeechAttribute.class);
@@ -190,8 +172,8 @@ public final class JapaneseTokenizer extends Tokenizer {
/**
* Create a new JapaneseTokenizer.
- * <p>
- * Uses the default AttributeFactory.
+ *
+ * <p>Uses the default AttributeFactory.
*
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
@@ -203,16 +185,26 @@ public final class JapaneseTokenizer extends Tokenizer {
/**
* Create a new JapaneseTokenizer.
- * <p>
- * Uses the default AttributeFactory.
+ *
+ * <p>Uses the default AttributeFactory.
*
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
- * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when
+ * tokenization mode is not NORMAL.
* @param mode tokenization mode.
*/
- public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuation, boolean discardCompoundToken, Mode mode) {
- this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, discardPunctuation, discardCompoundToken, mode);
+ public JapaneseTokenizer(
+ UserDictionary userDictionary,
+ boolean discardPunctuation,
+ boolean discardCompoundToken,
+ Mode mode) {
+ this(
+ DEFAULT_TOKEN_ATTRIBUTE_FACTORY,
+ userDictionary,
+ discardPunctuation,
+ discardCompoundToken,
+ mode);
}
/**
@@ -223,13 +215,20 @@ public final class JapaneseTokenizer extends Tokenizer {
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
* @param mode tokenization mode.
*/
- public JapaneseTokenizer
- (AttributeFactory factory, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
- this(factory,
- TokenInfoDictionary.getInstance(),
- UnknownDictionary.getInstance(),
- ConnectionCosts.getInstance(),
- userDictionary, discardPunctuation, true, mode);
+ public JapaneseTokenizer(
+ AttributeFactory factory,
+ UserDictionary userDictionary,
+ boolean discardPunctuation,
+ Mode mode) {
+ this(
+ factory,
+ TokenInfoDictionary.getInstance(),
+ UnknownDictionary.getInstance(),
+ ConnectionCosts.getInstance(),
+ userDictionary,
+ discardPunctuation,
+ true,
+ mode);
}
/**
@@ -238,22 +237,32 @@ public final class JapaneseTokenizer extends Tokenizer {
* @param factory the AttributeFactory to use
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
- * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when
+ * tokenization mode is not NORMAL.
* @param mode tokenization mode.
*/
- public JapaneseTokenizer
- (AttributeFactory factory, UserDictionary userDictionary, boolean discardPunctuation, boolean discardCompoundToken, Mode mode) {
- this(factory,
+ public JapaneseTokenizer(
+ AttributeFactory factory,
+ UserDictionary userDictionary,
+ boolean discardPunctuation,
+ boolean discardCompoundToken,
+ Mode mode) {
+ this(
+ factory,
TokenInfoDictionary.getInstance(),
UnknownDictionary.getInstance(),
ConnectionCosts.getInstance(),
- userDictionary, discardPunctuation, discardCompoundToken, mode);
+ userDictionary,
+ discardPunctuation,
+ discardCompoundToken,
+ mode);
}
/**
- * <p>Create a new JapaneseTokenizer, supplying a custom system dictionary and unknown dictionary.
- * This constructor provides an entry point for users that want to construct custom language models
- * that can be used as input to {@link org.apache.lucene.analysis.ja.util.DictionaryBuilder}.</p>
+ * Create a new JapaneseTokenizer, supplying a custom system dictionary and unknown dictionary.
+ * This constructor provides an entry point for users that want to construct custom language
+ * models that can be used as input to {@link
+ * org.apache.lucene.analysis.ja.util.DictionaryBuilder}.
*
* @param factory the AttributeFactory to use
* @param systemDictionary a custom known token dictionary
@@ -261,18 +270,20 @@ public final class JapaneseTokenizer extends Tokenizer {
* @param connectionCosts custom token transition costs
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
- * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when
+ * tokenization mode is not NORMAL.
* @param mode tokenization mode.
* @lucene.experimental
*/
- public JapaneseTokenizer(AttributeFactory factory,
- TokenInfoDictionary systemDictionary,
- UnknownDictionary unkDictionary,
- ConnectionCosts connectionCosts,
- UserDictionary userDictionary,
- boolean discardPunctuation,
- boolean discardCompoundToken,
- Mode mode) {
+ public JapaneseTokenizer(
+ AttributeFactory factory,
+ TokenInfoDictionary systemDictionary,
+ UnknownDictionary unkDictionary,
+ ConnectionCosts connectionCosts,
+ UserDictionary userDictionary,
+ boolean discardPunctuation,
+ boolean discardCompoundToken,
+ Mode mode) {
super(factory);
this.dictionary = systemDictionary;
this.fst = dictionary.getFST();
@@ -289,7 +300,7 @@ public final class JapaneseTokenizer extends Tokenizer {
userFSTReader = null;
}
this.discardPunctuation = discardPunctuation;
- switch(mode){
+ switch (mode) {
case SEARCH:
searchMode = true;
extendedMode = false;
@@ -317,8 +328,7 @@ public final class JapaneseTokenizer extends Tokenizer {
private GraphvizFormatter dotOut;
- /** Expert: set this to produce graphviz (dot) output of
- * the Viterbi lattice */
+ /** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */
public void setGraphvizFormatter(GraphvizFormatter dotOut) {
this.dotOut = dotOut;
}
@@ -380,7 +390,7 @@ public final class JapaneseTokenizer extends Tokenizer {
break;
}
}
- if (allKanji) { // Process only Kanji keywords
+ if (allKanji) { // Process only Kanji keywords
return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;
} else if (length > SEARCH_MODE_OTHER_LENGTH) {
return (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;
@@ -390,7 +400,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// Holds all back pointers arriving to this position:
- final static class Position {
+ static final class Position {
int pos;
@@ -413,11 +423,11 @@ public final class JapaneseTokenizer extends Tokenizer {
Type[] forwardType = new Type[8];
public void grow() {
- costs = ArrayUtil.grow(costs, 1+count);
- lastRightID = ArrayUtil.grow(lastRightID, 1+count);
- backPos = ArrayUtil.grow(backPos, 1+count);
- backIndex = ArrayUtil.grow(backIndex, 1+count);
- backID = ArrayUtil.grow(backID, 1+count);
+ costs = ArrayUtil.grow(costs, 1 + count);
+ lastRightID = ArrayUtil.grow(lastRightID, 1 + count);
+ backPos = ArrayUtil.grow(backPos, 1 + count);
+ backIndex = ArrayUtil.grow(backIndex, 1 + count);
+ backID = ArrayUtil.grow(backID, 1 + count);
// NOTE: sneaky: grow separately because
// ArrayUtil.grow will otherwise pick a different
@@ -428,9 +438,9 @@ public final class JapaneseTokenizer extends Tokenizer {
}
public void growForward() {
- forwardPos = ArrayUtil.grow(forwardPos, 1+forwardCount);
- forwardID = ArrayUtil.grow(forwardID, 1+forwardCount);
- forwardIndex = ArrayUtil.grow(forwardIndex, 1+forwardCount);
+ forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);
+ forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);
+ forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);
// NOTE: sneaky: grow separately because
// ArrayUtil.grow will otherwise pick a different
@@ -440,7 +450,8 @@ public final class JapaneseTokenizer extends Tokenizer {
forwardType = newForwardType;
}
- public void add(int cost, int lastRightID, int backPos, int backIndex, int backID, Type backType) {
+ public void add(
+ int cost, int lastRightID, int backPos, int backIndex, int backID, Type backType) {
// NOTE: this isn't quite a true Viterbi search,
// because we should check if lastRightID is
// already present here, and only update if the new
@@ -475,22 +486,37 @@ public final class JapaneseTokenizer extends Tokenizer {
public void reset() {
count = 0;
// forwardCount naturally resets after it runs:
- assert forwardCount == 0: "pos=" + pos + " forwardCount=" + forwardCount;
+ assert forwardCount == 0 : "pos=" + pos + " forwardCount=" + forwardCount;
}
}
- private void add(Dictionary dict, Position fromPosData, int endPos, int wordID, Type type, boolean addPenalty) throws IOException {
+ private void add(
+ Dictionary dict, Position fromPosData, int endPos, int wordID, Type type, boolean addPenalty)
+ throws IOException {
final int wordCost = dict.getWordCost(wordID);
final int leftID = dict.getLeftId(wordID);
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
assert fromPosData.count > 0;
- for(int idx=0;idx<fromPosData.count;idx++) {
+ for (int idx = 0; idx < fromPosData.count; idx++) {
// Cost is path cost so far, plus word cost (added at
// end of loop), plus bigram cost:
final int cost = fromPosData.costs[idx] + costs.get(fromPosData.lastRightID[idx], leftID);
if (VERBOSE) {
- System.out.println(" fromIDX=" + idx + ": cost=" + cost + " (prevCost=" + fromPosData.costs[idx] + " wordCost=" + wordCost + " bgCost=" + costs.get(fromPosData.lastRightID[idx], leftID) + " leftID=" + leftID + ")");
+ System.out.println(
+ " fromIDX="
+ + idx
+ + ": cost="
+ + cost
+ + " (prevCost="
+ + fromPosData.costs[idx]
+ + " wordCost="
+ + wordCost
+ + " bgCost="
+ + costs.get(fromPosData.lastRightID[idx], leftID)
+ + " leftID="
+ + leftID
+ + ")");
}
if (cost < leastCost) {
leastCost = cost;
@@ -504,20 +530,33 @@ public final class JapaneseTokenizer extends Tokenizer {
leastCost += wordCost;
if (VERBOSE) {
- System.out.println(" + cost=" + leastCost + " wordID=" + wordID + " leftID=" + leftID + " leastIDX=" + leastIDX + " toPos=" + endPos + " toPos.idx=" + positions.get(endPos).count);
+ System.out.println(
+ " + cost="
+ + leastCost
+ + " wordID="
+ + wordID
+ + " leftID="
+ + leftID
+ + " leastIDX="
+ + leastIDX
+ + " toPos="
+ + endPos
+ + " toPos.idx="
+ + positions.get(endPos).count);
}
if (addPenalty && type != Type.USER) {
final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
if (VERBOSE) {
if (penalty > 0) {
- System.out.println(" + penalty=" + penalty + " cost=" + (leastCost+penalty));
+ System.out.println(" + penalty=" + penalty + " cost=" + (leastCost + penalty));
}
}
leastCost += penalty;
}
- //positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX, wordID, type);
+ // positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX,
+ // wordID, type);
assert leftID == dict.getRightId(wordID);
positions.get(endPos).add(leastCost, leftID, fromPosData.pos, leastIDX, wordID, type);
}
@@ -538,15 +577,16 @@ public final class JapaneseTokenizer extends Tokenizer {
parse();
}
- final Token token = pending.remove(pending.size()-1);
+ final Token token = pending.remove(pending.size() - 1);
int position = token.getPosition();
int length = token.getLength();
clearAttributes();
assert length > 0;
- //System.out.println("off=" + token.getOffset() + " len=" + length + " vs " + token.getSurfaceForm().length);
+ // System.out.println("off=" + token.getOffset() + " len=" + length + " vs " +
+ // token.getSurfaceForm().length);
termAtt.copyBuffer(token.getSurfaceForm(), token.getOffset(), length);
- offsetAtt.setOffset(correctOffset(position), correctOffset(position+length));
+ offsetAtt.setOffset(correctOffset(position), correctOffset(position + length));
basicFormAtt.setToken(token);
posAtt.setToken(token);
readingAtt.setToken(token);
@@ -578,7 +618,7 @@ public final class JapaneseTokenizer extends Tokenizer {
private Position[] positions = new Position[8];
public WrappedPositionArray() {
- for(int i=0;i<positions.length;i++) {
+ for (int i = 0; i < positions.length; i++) {
positions[i] = new Position();
}
}
@@ -595,7 +635,7 @@ public final class JapaneseTokenizer extends Tokenizer {
public void reset() {
nextWrite--;
- while(count > 0) {
+ while (count > 0) {
if (nextWrite == -1) {
nextWrite = positions.length - 1;
}
@@ -607,18 +647,20 @@ public final class JapaneseTokenizer extends Tokenizer {
count = 0;
}
- /** Get Position instance for this absolute position;
- * this is allowed to be arbitrarily far "in the
- * future" but cannot be before the last freeBefore. */
+ /**
+ * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in
+ * the future" but cannot be before the last freeBefore.
+ */
public Position get(int pos) {
- while(pos >= nextPos) {
- //System.out.println("count=" + count + " vs len=" + positions.length);
+ while (pos >= nextPos) {
+ // System.out.println("count=" + count + " vs len=" + positions.length);
if (count == positions.length) {
- Position[] newPositions = new Position[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- //System.out.println("grow positions " + newPositions.length);
- System.arraycopy(positions, nextWrite, newPositions, 0, positions.length-nextWrite);
- System.arraycopy(positions, 0, newPositions, positions.length-nextWrite, nextWrite);
- for(int i=positions.length;i<newPositions.length;i++) {
+ Position[] newPositions =
+ new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ // System.out.println("grow positions " + newPositions.length);
+ System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);
+ System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);
+ for (int i = positions.length; i < newPositions.length; i++) {
newPositions[i] = new Position();
}
nextWrite = positions.length;
@@ -663,11 +705,11 @@ public final class JapaneseTokenizer extends Tokenizer {
if (index < 0) {
index += positions.length;
}
- for(int i=0;i<toFree;i++) {
+ for (int i = 0; i < toFree; i++) {
if (index == positions.length) {
index = 0;
}
- //System.out.println(" fb idx=" + index);
+ // System.out.println(" fb idx=" + index);
positions[index].reset();
index++;
}
@@ -699,7 +741,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
final Position posData = positions.get(pos);
- final boolean isFrontier = positions.getNextPos() == pos+1;
+ final boolean isFrontier = positions.getNextPos() == pos + 1;
if (posData.count == 0) {
// No arcs arrive here; move to next position:
@@ -751,10 +793,10 @@ public final class JapaneseTokenizer extends Tokenizer {
int leastIDX = -1;
int leastCost = Integer.MAX_VALUE;
Position leastPosData = null;
- for(int pos2=pos;pos2<positions.getNextPos();pos2++) {
+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
final Position posData2 = positions.get(pos2);
- for(int idx=0;idx<posData2.count;idx++) {
- //System.out.println(" idx=" + idx + " cost=" + cost);
+ for (int idx = 0; idx < posData2.count; idx++) {
+ // System.out.println(" idx=" + idx + " cost=" + cost);
final int cost = posData2.costs[idx];
if (cost < leastCost) {
leastCost = cost;
@@ -772,7 +814,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// Second pass: prune all but the best path:
- for(int pos2=pos;pos2<positions.getNextPos();pos2++) {
+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
final Position posData2 = positions.get(pos2);
if (posData2 != leastPosData) {
posData2.reset();
@@ -813,7 +855,13 @@ public final class JapaneseTokenizer extends Tokenizer {
}
if (VERBOSE) {
- System.out.println("\n extend @ pos=" + pos + " char=" + (char) buffer.get(pos) + " hex=" + Integer.toHexString(buffer.get(pos)));
+ System.out.println(
+ "\n extend @ pos="
+ + pos
+ + " char="
+ + (char) buffer.get(pos)
+ + " hex="
+ + Integer.toHexString(buffer.get(pos)));
}
if (VERBOSE) {
@@ -826,7 +874,7 @@ public final class JapaneseTokenizer extends Tokenizer {
if (userFST != null) {
userFST.getFirstArc(arc);
int output = 0;
- for(int posAhead=posData.pos;;posAhead++) {
+ for (int posAhead = posData.pos; ; posAhead++) {
final int ch = buffer.get(posAhead);
if (ch == -1) {
break;
@@ -837,9 +885,19 @@ public final class JapaneseTokenizer extends Tokenizer {
output += arc.output().intValue();
if (arc.isFinal()) {
if (VERBOSE) {
- System.out.println(" USER word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1));
+ System.out.println(
+ " USER word "
+ + new String(buffer.get(pos, posAhead - pos + 1))
+ + " toPos="
+ + (posAhead + 1));
}
- add(userDictionary, posData, posAhead+1, output + arc.nextFinalOutput().intValue(), Type.USER, false);
+ add(
+ userDictionary,
+ posData,
+ posAhead + 1,
+ output + arc.nextFinalOutput().intValue(),
+ Type.USER,
+ false);
anyMatches = true;
}
}
@@ -854,12 +912,12 @@ public final class JapaneseTokenizer extends Tokenizer {
fst.getFirstArc(arc);
int output = 0;
- for(int posAhead=posData.pos;;posAhead++) {
+ for (int posAhead = posData.pos; ; posAhead++) {
final int ch = buffer.get(posAhead);
if (ch == -1) {
break;
}
- //System.out.println(" match " + (char) ch + " posAhead=" + posAhead);
+ // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);
if (fst.findTargetArc(ch, arc, arc, posAhead == posData.pos, fstReader) == null) {
break;
@@ -876,10 +934,23 @@ public final class JapaneseTokenizer extends Tokenizer {
if (arc.isFinal()) {
dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);
if (VERBOSE) {
- System.out.println(" KNOWN word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1) + " " + wordIdRef.length + " wordIDs");
+ System.out.println(
+ " KNOWN word "
+ + new String(buffer.get(pos, posAhead - pos + 1))
+ + " toPos="
+ + (posAhead + 1)
+ + " "
+ + wordIdRef.length
+ + " wordIDs");
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(dictionary, posData, posAhead+1, wordIdRef.ints[wordIdRef.offset + ofs], Type.KNOWN, false);
+ add(
+ dictionary,
+ posData,
+ posAhead + 1,
+ wordIdRef.ints[wordIdRef.offset + ofs],
+ Type.KNOWN,
+ false);
anyMatches = true;
}
}
@@ -905,15 +976,16 @@ public final class JapaneseTokenizer extends Tokenizer {
if (!characterDefinition.isGroup(firstCharacter)) {
unknownWordLength = 1;
} else {
- // Extract unknown word. Characters with the same character class are considered to be part of unknown word
+ // Extract unknown word. Characters with the same character class are considered to be
+ // part of unknown word
unknownWordLength = 1;
- for (int posAhead=pos+1;unknownWordLength<MAX_UNKNOWN_WORD_LENGTH;posAhead++) {
+ for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
final int ch = buffer.get(posAhead);
if (ch == -1) {
break;
}
- if (characterId == characterDefinition.getCharacterClass((char) ch) &&
- isPunctuation((char) ch) == isPunct) {
+ if (characterId == characterDefinition.getCharacterClass((char) ch)
+ && isPunctuation((char) ch) == isPunct) {
unknownWordLength++;
} else {
break;
@@ -921,12 +993,20 @@ public final class JapaneseTokenizer extends Tokenizer {
}
}
- unkDictionary.lookupWordIds(characterId, wordIdRef); // characters in input text are supposed to be the same
+ unkDictionary.lookupWordIds(
+ characterId, wordIdRef); // characters in input text are supposed to be the same
if (VERBOSE) {
- System.out.println(" UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
+ System.out.println(
+ " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(unkDictionary, posData, posData.pos + unknownWordLength, wordIdRef.ints[wordIdRef.offset + ofs], Type.UNKNOWN, false);
+ add(
+ unkDictionary,
+ posData,
+ posData.pos + unknownWordLength,
+ wordIdRef.ints[wordIdRef.offset + ofs],
+ Type.UNKNOWN,
+ false);
}
unknownWordEndIndex = posData.pos + unknownWordLength;
@@ -945,10 +1025,12 @@ public final class JapaneseTokenizer extends Tokenizer {
if (VERBOSE) {
System.out.println(" end: " + endPosData.count + " nodes");
}
- for(int idx=0;idx<endPosData.count;idx++) {
+ for (int idx = 0; idx < endPosData.count; idx++) {
// Add EOS cost:
final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);
- //System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" + endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ") backPos=" + endPosData.backPos[idx]);
+ // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +
+ // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")
+ // backPos=" + endPosData.backPos[idx]);
if (cost < leastCost) {
leastCost = cost;
leastIDX = idx;
@@ -976,25 +1058,30 @@ public final class JapaneseTokenizer extends Tokenizer {
private void pruneAndRescore(int startPos, int endPos, int bestStartIDX) throws IOException {
if (VERBOSE) {
- System.out.println(" pruneAndRescore startPos=" + startPos + " endPos=" + endPos + " bestStartIDX=" + bestStartIDX);
+ System.out.println(
+ " pruneAndRescore startPos="
+ + startPos
+ + " endPos="
+ + endPos
+ + " bestStartIDX="
+ + bestStartIDX);
}
// First pass: walk backwards, building up the forward
// arcs and pruning inadmissible arcs:
- for(int pos=endPos; pos > startPos; pos--) {
+ for (int pos = endPos; pos > startPos; pos--) {
final Position posData = positions.get(pos);
if (VERBOSE) {
System.out.println(" back pos=" + pos);
}
- for(int arcIDX=0;arcIDX<posData.count;arcIDX++) {
+ for (int arcIDX = 0; arcIDX < posData.count; arcIDX++) {
final int backPos = posData.backPos[arcIDX];
if (backPos >= startPos) {
// Keep this arc:
- //System.out.println(" keep backPos=" + backPos);
- positions.get(backPos).addForward(pos,
- arcIDX,
- posData.backID[arcIDX],
- posData.backType[arcIDX]);
+ // System.out.println(" keep backPos=" + backPos);
+ positions
+ .get(backPos)
+ .addForward(pos, arcIDX, posData.backID[arcIDX], posData.backType[arcIDX]);
} else {
if (VERBOSE) {
System.out.println(" prune");
@@ -1007,7 +1094,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// Second pass: walk forward, re-scoring:
- for(int pos=startPos; pos < endPos; pos++) {
+ for (int pos = startPos; pos < endPos; pos++) {
final Position posData = positions.get(pos);
if (VERBOSE) {
System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);
@@ -1030,37 +1117,56 @@ public final class JapaneseTokenizer extends Tokenizer {
if (startPos == 0) {
rightID = 0;
} else {
- rightID = getDict(posData.backType[bestStartIDX]).getRightId(posData.backID[bestStartIDX]);
+ rightID =
+ getDict(posData.backType[bestStartIDX]).getRightId(posData.backID[bestStartIDX]);
}
final int pathCost = posData.costs[bestStartIDX];
- for(int forwardArcIDX=0;forwardArcIDX<posData.forwardCount;forwardArcIDX++) {
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
final Type forwardType = posData.forwardType[forwardArcIDX];
final Dictionary dict2 = getDict(forwardType);
final int wordID = posData.forwardID[forwardArcIDX];
final int toPos = posData.forwardPos[forwardArcIDX];
- final int newCost = pathCost + dict2.getWordCost(wordID) +
- costs.get(rightID, dict2.getLeftId(wordID)) +
- computePenalty(pos, toPos-pos);
+ final int newCost =
+ pathCost
+ + dict2.getWordCost(wordID)
+ + costs.get(rightID, dict2.getLeftId(wordID))
+ + computePenalty(pos, toPos - pos);
if (VERBOSE) {
- System.out.println(" + " + forwardType + " word " + new String(buffer.get(pos, toPos-pos)) + " toPos=" + toPos + " cost=" + newCost + " penalty=" + computePenalty(pos, toPos-pos) + " toPos.idx=" + positions.get(toPos).count);
+ System.out.println(
+ " + "
+ + forwardType
+ + " word "
+ + new String(buffer.get(pos, toPos - pos))
+ + " toPos="
+ + toPos
+ + " cost="
+ + newCost
+ + " penalty="
+ + computePenalty(pos, toPos - pos)
+ + " toPos.idx="
+ + positions.get(toPos).count);
}
- positions.get(toPos).add(newCost,
- dict2.getRightId(wordID),
- pos,
- bestStartIDX,
- wordID,
- forwardType);
+ positions
+ .get(toPos)
+ .add(newCost, dict2.getRightId(wordID), pos, bestStartIDX, wordID, forwardType);
}
} else {
// On non-initial positions, we maximize score
// across all arriving lastRightIDs:
- for(int forwardArcIDX=0;forwardArcIDX<posData.forwardCount;forwardArcIDX++) {
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
final Type forwardType = posData.forwardType[forwardArcIDX];
final int toPos = posData.forwardPos[forwardArcIDX];
if (VERBOSE) {
- System.out.println(" + " + forwardType + " word " + new String(buffer.get(pos, toPos-pos)) + " toPos=" + toPos);
+ System.out.println(
+ " + "
+ + forwardType
+ + " word "
+ + new String(buffer.get(pos, toPos - pos))
+ + " toPos="
+ + toPos);
}
- add(getDict(forwardType),
+ add(
+ getDict(forwardType),
posData,
toPos,
posData.forwardID[forwardArcIDX],
@@ -1073,7 +1179,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// yet another lattice data structure
- private final static class Lattice {
+ private static final class Lattice {
char[] fragment;
EnumMap<Type, Dictionary> dictionaryMap;
boolean useEOS;
@@ -1157,9 +1263,13 @@ public final class JapaneseTokenizer extends Tokenizer {
private int addNode(Type dicType, int wordID, int left, int right) {
if (VERBOSE) {
- System.out.printf("DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
- dicType.toString(), wordID, left, right,
- left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
+ System.out.printf(
+ "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
+ dicType.toString(),
+ wordID,
+ left,
+ right,
+ left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
}
assert nodeCount < capacity;
assert left == -1 || right == -1 || left < right;
@@ -1190,8 +1300,9 @@ public final class JapaneseTokenizer extends Tokenizer {
}
if (VERBOSE) {
- System.out.printf("DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
- nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
+ System.out.printf(
+ "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
+ nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
}
nodeLeft[node] = left;
@@ -1222,10 +1333,13 @@ public final class JapaneseTokenizer extends Tokenizer {
return count;
}
-
- void setup(char[] fragment,
- EnumMap<Type, Dictionary> dictionaryMap,
- WrappedPositionArray positions, int prevOffset, int endOffset, boolean useEOS) {
+ void setup(
+ char[] fragment,
+ EnumMap<Type, Dictionary> dictionaryMap,
+ WrappedPositionArray positions,
+ int prevOffset,
+ int endOffset,
+ boolean useEOS) {
assert positions.get(prevOffset).count == 1;
if (VERBOSE) {
System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);
@@ -1291,7 +1405,10 @@ public final class JapaneseTokenizer extends Tokenizer {
int leastCost = Integer.MAX_VALUE;
for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {
if (0 <= nodeMark[leftNode]) {
- int cost = nodeLeftCost[leftNode] + nodeWordCost[leftNode] + connectionCost(costs, leftNode, node);
+ int cost =
+ nodeLeftCost[leftNode]
+ + nodeWordCost[leftNode]
+ + connectionCost(costs, leftNode, node);
if (cost < leastCost) {
leastCost = cost;
leastNode = leftNode;
@@ -1302,8 +1419,9 @@ public final class JapaneseTokenizer extends Tokenizer {
nodeLeftNode[node] = leastNode;
nodeLeftCost[node] = leastCost;
if (VERBOSE) {
- System.out.printf("DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
- node, nodeLeftNode[node], nodeLeftCost[node]);
+ System.out.printf(
+ "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
+ node, nodeLeftNode[node], nodeLeftCost[node]);
}
}
}
@@ -1316,9 +1434,14 @@ public final class JapaneseTokenizer extends Tokenizer {
if (0 <= nodeMark[node]) {
int leastNode = -1;
int leastCost = Integer.MAX_VALUE;
- for (int rightNode = lRoot[index]; 0 <= rightNode; rightNode = nodeLeftChain[rightNode]) {
+ for (int rightNode = lRoot[index];
+ 0 <= rightNode;
+ rightNode = nodeLeftChain[rightNode]) {
if (0 <= nodeMark[rightNode]) {
- int cost = nodeRightCost[rightNode] + nodeWordCost[rightNode] + connectionCost(costs, node, rightNode);
+ int cost =
+ nodeRightCost[rightNode]
+ + nodeWordCost[rightNode]
+ + connectionCost(costs, node, rightNode);
if (cost < leastCost) {
leastCost = cost;
leastNode = rightNode;
@@ -1329,8 +1452,9 @@ public final class JapaneseTokenizer extends Tokenizer {
nodeRightNode[node] = leastNode;
nodeRightCost[node] = leastCost;
if (VERBOSE) {
- System.out.printf("DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
- node, nodeRightNode[node], nodeRightCost[node]);
+ System.out.printf(
+ "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
+ node, nodeRightNode[node], nodeRightCost[node]);
}
}
}
@@ -1375,7 +1499,8 @@ public final class JapaneseTokenizer extends Tokenizer {
leastRight = nodeRight[node];
list.clear();
list.add(node);
- } else if (cost == leastCost && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
+ } else if (cost == leastCost
+ && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
list.add(node);
}
}
@@ -1408,8 +1533,9 @@ public final class JapaneseTokenizer extends Tokenizer {
void debugPrint() {
if (VERBOSE) {
for (int node = 0; node < nodeCount; ++node) {
- System.out.printf("DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
- node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
+ System.out.printf(
+ "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
+ node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
}
}
}
@@ -1428,36 +1554,42 @@ public final class JapaneseTokenizer extends Tokenizer {
// Expand the phraseID we recorded into the actual segmentation:
final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);
int wordID = wordIDAndLength[0];
- pending.add(new Token(wordID,
- fragment,
- left,
- right - left,
- Type.USER,
- lattice.rootBase + left,
- userDictionary));
+ pending.add(
+ new Token(
+ wordID,
+ fragment,
+ left,
+ right - left,
+ Type.USER,
+ lattice.rootBase + left,
+ userDictionary));
// Output compound
int current = 0;
for (int j = 1; j < wordIDAndLength.length; j++) {
final int len = wordIDAndLength[j];
if (len < right - left) {
- pending.add(new Token(wordID + j - 1,
- fragment,
- current + left,
- len,
- Type.USER,
- lattice.rootBase + current + left,
- userDictionary));
+ pending.add(
+ new Token(
+ wordID + j - 1,
+ fragment,
+ current + left,
+ len,
+ Type.USER,
+ lattice.rootBase + current + left,
+ userDictionary));
}
current += len;
}
} else {
- pending.add(new Token(lattice.nodeWordID[node],
- fragment,
- left,
- right - left,
- type,
- lattice.rootBase + left,
- getDict(type)));
+ pending.add(
+ new Token(
+ lattice.nodeWordID[node],
+ fragment,
+ left,
+ right - left,
+ type,
+ lattice.rootBase + left,
+ getDict(type)));
}
}
}
@@ -1466,25 +1598,26 @@ public final class JapaneseTokenizer extends Tokenizer {
private void fixupPendingList() {
// Sort for removing same tokens.
// USER token should be ahead from normal one.
- Collections.sort(pending,
- new Comparator<Token>() {
- @Override
- public int compare(Token a, Token b) {
- int aOff = a.getOffset();
- int bOff = b.getOffset();
- if (aOff != bOff) {
- return aOff - bOff;
- }
- int aLen = a.getLength();
- int bLen = b.getLength();
- if (aLen != bLen) {
- return aLen - bLen;
- }
- // order of Type is KNOWN, UNKNOWN, USER,
- // so we use reversed comparison here.
- return b.getType().ordinal() - a.getType().ordinal();
- }
- });
+ Collections.sort(
+ pending,
+ new Comparator<Token>() {
+ @Override
+ public int compare(Token a, Token b) {
+ int aOff = a.getOffset();
+ int bOff = b.getOffset();
+ if (aOff != bOff) {
+ return aOff - bOff;
+ }
+ int aLen = a.getLength();
+ int bLen = b.getLength();
+ if (aLen != bLen) {
+ return aLen - bLen;
+ }
+ // order of Type is KNOWN, UNKNOWN, USER,
+ // so we use reversed comparison here.
+ return b.getType().ordinal() - a.getType().ordinal();
+ }
+ });
// Remove same token.
for (int i = 1; i < pending.size(); ++i) {
@@ -1569,7 +1702,8 @@ public final class JapaneseTokenizer extends Tokenizer {
try {
maxDelta = Math.max(maxDelta, probeDelta(pair[0], pair[1]));
} catch (IOException e) {
- throw new RuntimeException("Internal error calculating best costs from examples. Got ", e);
+ throw new RuntimeException(
+ "Internal error calculating best costs from examples. Got ", e);
}
}
}
@@ -1602,7 +1736,7 @@ public final class JapaneseTokenizer extends Tokenizer {
registerNode(node, fragment);
}
- for (int n = 2;; ++n) {
+ for (int n = 2; ; ++n) {
List<Integer> nbest = lattice.nBestNodeList(n);
if (nbest.isEmpty()) {
break;
@@ -1631,10 +1765,20 @@ public final class JapaneseTokenizer extends Tokenizer {
final int endPos = endPosData.pos;
if (VERBOSE) {
- System.out.println("\n backtrace: endPos=" + endPos + " pos=" + pos + "; " + (pos - lastBackTracePos) + " characters; last=" + lastBackTracePos + " cost=" + endPosData.costs[fromIDX]);
+ System.out.println(
+ "\n backtrace: endPos="
+ + endPos
+ + " pos="
+ + pos
+ + "; "
+ + (pos - lastBackTracePos)
+ + " characters; last="
+ + lastBackTracePos
+ + " cost="
+ + endPosData.costs[fromIDX]);
}
- final char[] fragment = buffer.get(lastBackTracePos, endPos-lastBackTracePos);
+ final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
if (dotOut != null) {
dotOut.onBacktrace(this, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);
@@ -1659,12 +1803,13 @@ public final class JapaneseTokenizer extends Tokenizer {
// incrementToken.
while (pos > lastBackTracePos) {
- //System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
+ // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
final Position posData = positions.get(pos);
assert bestIDX < posData.count;
int backPos = posData.backPos[bestIDX];
- assert backPos >= lastBackTracePos: "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
+ assert backPos >= lastBackTracePos
+ : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
int length = pos - backPos;
Type backType = posData.backType[bestIDX];
int backID = posData.backID[bestIDX];
@@ -1678,13 +1823,27 @@ public final class JapaneseTokenizer extends Tokenizer {
// alternate back trace with cost below that
// threshold, we pursue it instead (but also output
// the long token).
- //System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);
+ // System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);
- final int penalty = computeSecondBestThreshold(backPos, pos-backPos);
+ final int penalty = computeSecondBestThreshold(backPos, pos - backPos);
if (penalty > 0) {
if (VERBOSE) {
- System.out.println(" compound=" + new String(buffer.get(backPos, pos-backPos)) + " backPos=" + backPos + " pos=" + pos + " penalty=" + penalty + " cost=" + posData.costs[bestIDX] + " bestIDX=" + bestIDX + " lastLeftID=" + lastLeftWordID);
+ System.out.println(
+ " compound="
+ + new String(buffer.get(backPos, pos - backPos))
+ + " backPos="
+ + backPos
+ + " pos="
+ + pos
+ + " penalty="
+ + penalty
+ + " cost="
+ + posData.costs[bestIDX]
+ + " bestIDX="
+ + bestIDX
+ + " lastLeftID="
+ + lastLeftWordID);
}
// Use the penalty to set maxCost on the 2nd best
@@ -1695,35 +1854,45 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// Now, prune all too-long tokens from the graph:
- pruneAndRescore(backPos, pos,
- posData.backIndex[bestIDX]);
+ pruneAndRescore(backPos, pos, posData.backIndex[bestIDX]);
// Finally, find 2nd best back-trace and resume
// backtrace there:
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
- for(int idx=0;idx<posData.count;idx++) {
+ for (int idx = 0; idx < posData.count; idx++) {
int cost = posData.costs[idx];
- //System.out.println(" idx=" + idx + " prevCost=" + cost);
+ // System.out.println(" idx=" + idx + " prevCost=" + cost);
if (lastLeftWordID != -1) {
- cost += costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
- lastLeftWordID);
- //System.out.println(" += bgCost=" + costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
- //lastLeftWordID) + " -> " + cost);
+ cost +=
+ costs.get(
+ getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
+ lastLeftWordID);
+ // System.out.println(" += bgCost=" +
+ // costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
+ // lastLeftWordID) + " -> " + cost);
}
- //System.out.println("penalty " + posData.backPos[idx] + " to " + pos);
- //cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);
+ // System.out.println("penalty " + posData.backPos[idx] + " to " + pos);
+ // cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);
if (cost < leastCost) {
- //System.out.println(" ** ");
+ // System.out.println(" ** ");
leastCost = cost;
leastIDX = idx;
}
}
- //System.out.println(" leastIDX=" + leastIDX);
+ // System.out.println(" leastIDX=" + leastIDX);
if (VERBOSE) {
- System.out.println(" afterPrune: " + posData.count + " arcs arriving; leastCost=" + leastCost + " vs threshold=" + maxCost + " lastLeftWordID=" + lastLeftWordID);
+ System.out.println(
+ " afterPrune: "
+ + posData.count
+ + " arcs arriving; leastCost="
+ + leastCost
+ + " vs threshold="
+ + maxCost
+ + " lastLeftWordID="
+ + lastLeftWordID);
}
if (leastIDX != -1 && leastCost <= maxCost && posData.backPos[leastIDX] != backPos) {
@@ -1732,13 +1901,15 @@ public final class JapaneseTokenizer extends Tokenizer {
// Save the current compound token, to output when
// this alternate path joins back:
- altToken = new Token(backID,
- fragment,
- backPos - lastBackTracePos,
- length,
- backType,
- backPos,
- getDict(backType));
+ altToken =
+ new Token(
+ backID,
+ fragment,
+ backPos - lastBackTracePos,
+ length,
+ backType,
+ backPos,
+ getDict(backType));
// Redirect our backtrace to 2nd best:
bestIDX = leastIDX;
@@ -1749,13 +1920,13 @@ public final class JapaneseTokenizer extends Tokenizer {
backType = posData.backType[bestIDX];
backID = posData.backID[bestIDX];
backCount = 0;
- //System.out.println(" do alt token!");
+ // System.out.println(" do alt token!");
} else {
// I think in theory it's possible there is no
// 2nd best path, which is fine; in this case we
// only output the compound token:
- //System.out.println(" no alt token! bestIDX=" + bestIDX);
+ // System.out.println(" no alt token! bestIDX=" + bestIDX);
}
}
}
@@ -1806,18 +1977,20 @@ public final class JapaneseTokenizer extends Tokenizer {
final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);
int wordID = wordIDAndLength[0];
int current = 0;
- for(int j=1; j < wordIDAndLength.length; j++) {
+ for (int j = 1; j < wordIDAndLength.length; j++) {
final int len = wordIDAndLength[j];
- //System.out.println(" add user: len=" + len);
- pending.add(new Token(wordID+j-1,
- fragment,
- current + offset,
- len,
- Type.USER,
- current + backPos,
- dict));
+ // System.out.println(" add user: len=" + len);
+ pending.add(
+ new Token(
+ wordID + j - 1,
+ fragment,
+ current + offset,
+ len,
+ Type.USER,
+ current + backPos,
+ dict));
if (VERBOSE) {
- System.out.println(" add USER token=" + pending.get(pending.size()-1));
+ System.out.println(" add USER token=" + pending.get(pending.size() - 1));
}
current += len;
}
@@ -1825,52 +1998,49 @@ public final class JapaneseTokenizer extends Tokenizer {
// Reverse the tokens we just added, because when we
// serve them up from incrementToken we serve in
// reverse:
- Collections.reverse(pending.subList(pending.size() - (wordIDAndLength.length - 1),
- pending.size()));
+ Collections.reverse(
+ pending.subList(pending.size() - (wordIDAndLength.length - 1), pending.size()));
- backCount += wordIDAndLength.length-1;
+ backCount += wordIDAndLength.length - 1;
} else {
if (extendedMode && backType == Type.UNKNOWN) {
// In EXTENDED mode we convert unknown word into
// unigrams:
int unigramTokenCount = 0;
- for(int i=length-1;i>=0;i--) {
+ for (int i = length - 1; i >= 0; i--) {
int charLen = 1;
- if (i > 0 && Character.isLowSurrogate(fragment[offset+i])) {
+ if (i > 0 && Character.isLowSurrogate(fragment[offset + i])) {
i--;
charLen = 2;
}
- //System.out.println(" extended tok offset="
- //+ (offset + i));
- if (!discardPunctuation || !isPunctuation(fragment[offset+i])) {
- pending.add(new Token(CharacterDefinition.NGRAM,
- fragment,
- offset + i,
- charLen,
- Type.UNKNOWN,
- backPos + i,
- unkDictionary));
+ // System.out.println(" extended tok offset="
+ // + (offset + i));
+ if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {
+ pending.add(
+ new Token(
+ CharacterDefinition.NGRAM,
+ fragment,
+ offset + i,
+ charLen,
+ Type.UNKNOWN,
+ backPos + i,
+ unkDictionary));
unigramTokenCount++;
}
}
backCount += unigramTokenCount;
} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
- pending.add(new Token(backID,
- fragment,
- offset,
- length,
- backType,
- backPos,
- dict));
+ pending.add(new Token(backID, fragment, offset, length, backType, backPos, dict));
if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size()-1));
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
}
backCount++;
} else {
if (VERBOSE) {
- System.out.println(" skip punctuation token=" + new String(fragment, offset, length));
+ System.out.println(
+ " skip punctuation token=" + new String(fragment, offset, length));
}
}
}
@@ -1896,7 +2066,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
private static boolean isPunctuation(char ch) {
- switch(Character.getType(ch)) {
+ switch (Character.getType(ch)) {
case Character.SPACE_SEPARATOR:
case Character.LINE_SEPARATOR:
case Character.PARAGRAPH_SEPARATOR:
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
index 884fc2a..76105a9 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.analysis.ja;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -26,10 +25,9 @@ import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.util.Locale;
import java.util.Map;
-
+import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
-import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ResourceLoader;
@@ -37,6 +35,7 @@ import org.apache.lucene.util.ResourceLoaderAware;
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapaneseTokenizer}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
@@ -51,32 +50,28 @@ import org.apache.lucene.util.ResourceLoaderAware;
* </analyzer>
* </fieldType>
* </pre>
- * <p>
- * Additional expert user parameters nBestCost and nBestExamples can be
- * used to include additional searchable tokens that those most likely
- * according to the statistical model. A typical use-case for this is to
- * improve recall and make segmentation more resilient to mistakes.
- * The feature can also be used to get a decompounding effect.
- * <p>
- * The nBestCost parameter specifies an additional Viterbi cost, and
- * when used, JapaneseTokenizer will include all tokens in Viterbi paths
- * that are within the nBestCost value of the best path.
- * <p>
- * Finding a good value for nBestCost can be difficult to do by hand. The
- * nBestExamples parameter can be used to find an nBestCost value based on
- * examples with desired segmentation outcomes.
- * <p>
- * For example, a value of /箱根山-箱根/成田空港-成田/ indicates that in
- * the texts, 箱根山 (Mt. Hakone) and 成田空港 (Narita Airport) we'd like
- * a cost that gives is us 箱根 (Hakone) and 成田 (Narita). Notice that
- * costs are estimated for each example individually, and the maximum
- * nBestCost found across all examples is used.
- * <p>
- * If both nBestCost and nBestExamples is used in a configuration,
- * the largest value of the two is used.
- * <p>
- * Parameters nBestCost and nBestExamples work with all tokenizer
- * modes, but it makes the most sense to use them with NORMAL mode.
+ *
+ * <p>Additional expert user parameters nBestCost and nBestExamples can be used to include
+ * additional searchable tokens that those most likely according to the statistical model. A typical
+ * use-case for this is to improve recall and make segmentation more resilient to mistakes. The
+ * feature can also be used to get a decompounding effect.
+ *
+ * <p>The nBestCost parameter specifies an additional Viterbi cost, and when used, JapaneseTokenizer
+ * will include all tokens in Viterbi paths that are within the nBestCost value of the best path.
+ *
+ * <p>Finding a good value for nBestCost can be difficult to do by hand. The nBestExamples parameter
+ * can be used to find an nBestCost value based on examples with desired segmentation outcomes.
+ *
+ * <p>For example, a value of /箱根山-箱根/成田空港-成田/ indicates that in the texts, 箱根山 (Mt. Hakone) and
+ * 成田空港 (Narita Airport) we'd like a cost that gives is us 箱根 (Hakone) and 成田 (Narita). Notice that
+ * costs are estimated for each example individually, and the maximum nBestCost found across all
+ * examples is used.
+ *
+ * <p>If both nBestCost and nBestExamples is used in a configuration, the largest value of the two
+ * is used.
+ *
+ * <p>Parameters nBestCost and nBestExamples work with all tokenizer modes, but it makes the most
+ * sense to use them with NORMAL mode.
*
* @since 3.6.0
* @lucene.spi {@value #NAME}
@@ -122,9 +117,11 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
private int nbestCost = -1;
/** Creates a new JapaneseTokenizerFactory */
- public JapaneseTokenizerFactory(Map<String,String> args) {
+ public JapaneseTokenizerFactory(Map<String, String> args) {
super(args);
- mode = Mode.valueOf(get(args, MODE, JapaneseTokenizer.DEFAULT_MODE.toString()).toUpperCase(Locale.ROOT));
+ mode =
+ Mode.valueOf(
+ get(args, MODE, JapaneseTokenizer.DEFAULT_MODE.toString()).toUpperCase(Locale.ROOT));
userDictionaryPath = args.remove(USER_DICT_PATH);
userDictionaryEncoding = args.remove(USER_DICT_ENCODING);
discardPunctuation = getBoolean(args, DISCARD_PUNCTUATION, true);
@@ -149,9 +146,11 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
if (encoding == null) {
encoding = IOUtils.UTF_8;
}
- CharsetDecoder decoder = Charset.forName(encoding).newDecoder()
- .onMalformedInput(CodingErrorAction.REPORT)
- .onUnmappableCharacter(CodingErrorAction.REPORT);
+ CharsetDecoder decoder =
+ Charset.forName(encoding)
+ .newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT);
Reader reader = new InputStreamReader(stream, decoder);
userDictionary = UserDictionary.open(reader);
}
@@ -162,7 +161,9 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
@Override
public JapaneseTokenizer create(AttributeFactory factory) {
- JapaneseTokenizer t = new JapaneseTokenizer(factory, userDictionary, discardPunctuation, discardCompoundToken, mode);
+ JapaneseTokenizer t =
+ new JapaneseTokenizer(
+ factory, userDictionary, discardPunctuation, discardCompoundToken, mode);
if (nbestExamples != null) {
nbestCost = Math.max(nbestCost, t.calcNBestCost(nbestExamples));
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
index 12f9221..5a10b50 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
@@ -16,13 +16,10 @@
*/
package org.apache.lucene.analysis.ja;
-
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
import org.apache.lucene.analysis.ja.dict.Dictionary;
-/**
- * Analyzed token with morphological data from its dictionary.
- */
+/** Analyzed token with morphological data from its dictionary. */
public class Token {
private final Dictionary dictionary;
@@ -37,7 +34,14 @@ public class Token {
private final Type type;
- public Token(int wordId, char[] surfaceForm, int offset, int length, Type type, int position, Dictionary dictionary) {
+ public Token(
+ int wordId,
+ char[] surfaceForm,
+ int offset,
+ int length,
+ Type type,
+ int position,
+ Dictionary dictionary) {
this.wordId = wordId;
this.surfaceForm = surfaceForm;
this.offset = offset;
@@ -49,83 +53,76 @@ public class Token {
@Override
public String toString() {
- return "Token(\"" + new String(surfaceForm, offset, length) + "\" pos=" + position + " length=" + length +
- " posLen=" + positionLength + " type=" + type + " wordId=" + wordId +
- " leftID=" + dictionary.getLeftId(wordId) + ")";
- }
-
- /**
- * @return surfaceForm
- */
+ return "Token(\""
+ + new String(surfaceForm, offset, length)
+ + "\" pos="
+ + position
+ + " length="
+ + length
+ + " posLen="
+ + positionLength
+ + " type="
+ + type
+ + " wordId="
+ + wordId
+ + " leftID="
+ + dictionary.getLeftId(wordId)
+ + ")";
+ }
+
+ /** @return surfaceForm */
public char[] getSurfaceForm() {
return surfaceForm;
}
- /**
- * @return offset into surfaceForm
- */
+ /** @return offset into surfaceForm */
public int getOffset() {
return offset;
}
- /**
- * @return length of surfaceForm
- */
+ /** @return length of surfaceForm */
public int getLength() {
return length;
}
- /**
- * @return surfaceForm as a String
- */
+ /** @return surfaceForm as a String */
public String getSurfaceFormString() {
return new String(surfaceForm, offset, length);
}
- /**
- * @return reading. null if token doesn't have reading.
- */
+ /** @return reading. null if token doesn't have reading. */
public String getReading() {
return dictionary.getReading(wordId, surfaceForm, offset, length);
}
- /**
- * @return pronunciation. null if token doesn't have pronunciation.
- */
+ /** @return pronunciation. null if token doesn't have pronunciation. */
public String getPronunciation() {
return dictionary.getPronunciation(wordId, surfaceForm, offset, length);
}
- /**
- * @return part of speech.
- */
+ /** @return part of speech. */
public String getPartOfSpeech() {
return dictionary.getPartOfSpeech(wordId);
}
- /**
- * @return inflection type or null
- */
+ /** @return inflection type or null */
public String getInflectionType() {
return dictionary.getInflectionType(wordId);
}
- /**
- * @return inflection form or null
- */
+ /** @return inflection form or null */
public String getInflectionForm() {
return dictionary.getInflectionForm(wordId);
}
- /**
- * @return base form or null if token is not inflected
- */
+ /** @return base form or null if token is not inflected */
public String getBaseForm() {
return dictionary.getBaseForm(wordId, surfaceForm, offset, length);
}
/**
* Returns the type of this token
+ *
* @return token type, not null
*/
public Type getType() {
@@ -134,6 +131,7 @@ public class Token {
/**
* Returns true if this token is known word
+ *
* @return true if this token is in standard dictionary. false if not.
*/
public boolean isKnown() {
@@ -142,6 +140,7 @@ public class Token {
/**
* Returns true if this token is unknown word
+ *
* @return true if this token is unknown word. false if not.
*/
public boolean isUnknown() {
@@ -150,6 +149,7 @@ public class Token {
/**
* Returns true if this token is defined in user dictionary
+ *
* @return true if this token is in user dictionary. false if not.
*/
public boolean isUser() {
@@ -158,6 +158,7 @@ public class Token {
/**
* Get index of this token in input text
+ *
* @return position of token
*/
public int getPosition() {
@@ -165,16 +166,17 @@ public class Token {
}
/**
- * Set the position length (in tokens) of this token. For normal
- * tokens this is 1; for compound tokens it's > 1.
+ * Set the position length (in tokens) of this token. For normal tokens this is 1; for compound
+ * tokens it's > 1.
*/
public void setPositionLength(int positionLength) {
this.positionLength = positionLength;
}
/**
- * Get the length (in tokens) of this token. For normal
- * tokens this is 1; for compound tokens it's > 1.
+ * Get the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's
+ * > 1.
+ *
* @return position length of token
*/
public int getPositionLength() {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
index 670a5a4..73fa7f3 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
@@ -16,45 +16,40 @@
*/
package org.apache.lucene.analysis.ja.dict;
-
import java.io.BufferedInputStream;
import java.io.EOFException;
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
-
+import java.nio.file.Files;
+import java.nio.file.Paths;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.IntsRef;
-/**
- * Base class for a binary-encoded in-memory dictionary.
- */
+/** Base class for a binary-encoded in-memory dictionary. */
public abstract class BinaryDictionary implements Dictionary {
-
- /**
- * Used to specify where (dictionary) resources get loaded from.
- */
+
+ /** Used to specify where (dictionary) resources get loaded from. */
public enum ResourceScheme {
- CLASSPATH, FILE
+ CLASSPATH,
+ FILE
}
public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";
public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";
public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";
-
+
public static final String DICT_HEADER = "kuromoji_dict";
public static final String TARGETMAP_HEADER = "kuromoji_dict_map";
public static final String POSDICT_HEADER = "kuromoji_dict_pos";
public static final int VERSION = 1;
-
+
private final ResourceScheme resourceScheme;
private final String resourcePath;
private final ByteBuffer buffer;
@@ -62,21 +57,23 @@ public abstract class BinaryDictionary implements Dictionary {
private final String[] posDict;
private final String[] inflTypeDict;
private final String[] inflFormDict;
-
+
protected BinaryDictionary() throws IOException {
this(ResourceScheme.CLASSPATH, null);
}
/**
* @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
- * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
- * this class's name as the path.
+ * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH
+ * scheme only, use this class's name as the path.
*/
- protected BinaryDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
+ protected BinaryDictionary(ResourceScheme resourceScheme, String resourcePath)
+ throws IOException {
this.resourceScheme = resourceScheme;
if (resourcePath == null) {
if (resourceScheme != ResourceScheme.CLASSPATH) {
- throw new IllegalArgumentException("resourcePath must be supplied with FILE resource scheme");
+ throw new IllegalArgumentException(
+ "resourcePath must be supplied with FILE resource scheme");
}
this.resourcePath = getClass().getName().replace('.', '/');
} else {
@@ -107,12 +104,17 @@ public abstract class BinaryDictionary implements Dictionary {
targetMap[ofs] = accum;
}
if (sourceId + 1 != targetMapOffsets.length)
- throw new IOException("targetMap file format broken; targetMap.length=" + targetMap.length
- + ", targetMapOffsets.length=" + targetMapOffsets.length
- + ", sourceId=" + sourceId);
+ throw new IOException(
+ "targetMap file format broken; targetMap.length="
+ + targetMap.length
+ + ", targetMapOffsets.length="
+ + targetMapOffsets.length
+ + ", sourceId="
+ + sourceId);
targetMapOffsets[sourceId] = targetMap.length;
- mapIS.close(); mapIS = null;
-
+ mapIS.close();
+ mapIS = null;
+
posIS = getResource(POSDICT_FILENAME_SUFFIX);
posIS = new BufferedInputStream(posIS);
in = new InputStreamDataInput(posIS);
@@ -133,8 +135,9 @@ public abstract class BinaryDictionary implements Dictionary {
inflFormDict[j] = null;
}
}
- posIS.close(); posIS = null;
-
+ posIS.close();
+ posIS = null;
+
dictIS = getResource(DICT_FILENAME_SUFFIX);
// no buffering here, as we load in one large buffer
in = new InputStreamDataInput(dictIS);
@@ -146,7 +149,8 @@ public abstract class BinaryDictionary implements Dictionary {
if (read != size) {
throw new EOFException("Cannot read whole dictionary");
}
- dictIS.close(); dictIS = null;
+ dictIS.close();
+ dictIS = null;
buffer = tmpBuffer.asReadOnlyBuffer();
success = true;
} finally {
@@ -156,7 +160,7 @@ public abstract class BinaryDictionary implements Dictionary {
IOUtils.closeWhileHandlingException(mapIS, posIS, dictIS);
}
}
-
+
this.targetMap = targetMap;
this.targetMapOffsets = targetMapOffsets;
this.posDict = posDict;
@@ -164,9 +168,9 @@ public abstract class BinaryDictionary implements Dictionary {
this.inflFormDict = inflFormDict;
this.buffer = buffer;
}
-
+
protected final InputStream getResource(String suffix) throws IOException {
- switch(resourceScheme) {
+ switch (resourceScheme) {
case CLASSPATH:
return getClassResource(resourcePath + suffix);
case FILE:
@@ -175,9 +179,10 @@ public abstract class BinaryDictionary implements Dictionary {
throw new IllegalStateException("unknown resource scheme " + resourceScheme);
}
}
-
- public static final InputStream getResource(ResourceScheme scheme, String path) throws IOException {
- switch(scheme) {
+
+ public static final InputStream getResource(ResourceScheme scheme, String path)
+ throws IOException {
+ switch (scheme) {
case CLASSPATH:
return getClassResource(path);
case FILE:
@@ -188,14 +193,16 @@ public abstract class BinaryDictionary implements Dictionary {
}
// util, reused by ConnectionCosts and CharacterDefinition
- public static final InputStream getClassResource(Class<?> clazz, String suffix) throws IOException {
+ public static final InputStream getClassResource(Class<?> clazz, String suffix)
+ throws IOException {
final InputStream is = clazz.getResourceAsStream(clazz.getSimpleName() + suffix);
if (is == null) {
- throw new FileNotFoundException("Not in classpath: " + clazz.getName().replace('.','/') + suffix);
+ throw new FileNotFoundException(
+ "Not in classpath: " + clazz.getName().replace('.', '/') + suffix);
}
return is;
}
-
+
private static InputStream getClassResource(String path) throws IOException {
final InputStream is = BinaryDictionary.class.getClassLoader().getResourceAsStream(path);
if (is == null) {
@@ -210,20 +217,20 @@ public abstract class BinaryDictionary implements Dictionary {
// targetMapOffsets always has one more entry pointing behind last:
ref.length = targetMapOffsets[sourceId + 1] - ref.offset;
}
-
+
@Override
public int getLeftId(int wordId) {
return (buffer.getShort(wordId) & 0xffff) >>> 3;
}
-
+
@Override
public int getRightId(int wordId) {
return (buffer.getShort(wordId) & 0xffff) >>> 3;
}
-
+
@Override
public int getWordCost(int wordId) {
- return buffer.getShort(wordId + 2); // Skip id
+ return buffer.getShort(wordId + 2); // Skip id
}
@Override
@@ -233,17 +240,17 @@ public abstract class BinaryDictionary implements Dictionary {
int data = buffer.get(offset++) & 0xff;
int prefix = data >>> 4;
int suffix = data & 0xF;
- char text[] = new char[prefix+suffix];
+ char text[] = new char[prefix + suffix];
System.arraycopy(surfaceForm, off, text, 0, prefix);
for (int i = 0; i < suffix; i++) {
- text[prefix+i] = buffer.getChar(offset + (i << 1));
+ text[prefix + i] = buffer.getChar(offset + (i << 1));
}
return new String(text);
} else {
return null;
}
}
-
+
@Override
public String getReading(int wordId, char surface[], int off, int len) {
if (hasReadingData(wordId)) {
@@ -254,9 +261,9 @@ public abstract class BinaryDictionary implements Dictionary {
// the reading is the surface form, with hiragana shifted to katakana
char text[] = new char[len];
for (int i = 0; i < len; i++) {
- char ch = surface[off+i];
+ char ch = surface[off + i];
if (ch > 0x3040 && ch < 0x3097) {
- text[i] = (char)(ch + 0x60);
+ text[i] = (char) (ch + 0x60);
} else {
text[i] = ch;
}
@@ -264,12 +271,12 @@ public abstract class BinaryDictionary implements Dictionary {
return new String(text);
}
}
-
+
@Override
public String getPartOfSpeech(int wordId) {
return posDict[getLeftId(wordId)];
}
-
+
@Override
public String getPronunciation(int wordId, char surface[], int off, int len) {
if (hasPronunciationData(wordId)) {
@@ -280,7 +287,7 @@ public abstract class BinaryDictionary implements Dictionary {
return getReading(wordId, surface, off, len); // same as the reading
}
}
-
... 112098 lines suppressed ...
[lucene-solr] 02/02: LUCENE-9570: code reformatting [record rev].
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 23e206d085c92c47b33f124eb85bfbbd5facbab5
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Sun Jan 3 21:46:35 2021 +0100
LUCENE-9570: code reformatting [record rev].
---
.git-blame-ignore-revs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index ad75695..91520ff 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -1,4 +1,5 @@
# LUCENE-9570 [google java formatter applied]
6faa4f98e04817bf27d02d96332904a29fb7db69
2d6ad2fee6dfd96388594f4de9b37c037efe8017
-8ef6a0da56878177ff8d6880c92e8f7d0321d076
\ No newline at end of file
+8ef6a0da56878177ff8d6880c92e8f7d0321d076
+7f8b7ffbcad2265b047a5e2195f76cc924028063