You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2020/12/28 11:27:03 UTC
[lucene-solr] branch master updated (409a0ac -> 4160ced)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 409a0ac LUCENE-9643: git revisions for .git-blame-ignore-revs need to be full sha1, not abbreviated. (#2167)
new 9af588e Correct extra * in javadoc.
new 8ef6a0d LUCENE-9570: code reformatting [partial].
new 4160ced LUCENE-9570: code reformatting [record rev].
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.git-blame-ignore-revs | 3 +-
gradle/generation/jflex.gradle | 46 +-
gradle/validation/spotless.gradle | 43 +-
.../lucene/analysis/bg/BulgarianAnalyzer.java | 76 +-
.../lucene/analysis/bg/BulgarianStemFilter.java | 24 +-
.../analysis/bg/BulgarianStemFilterFactory.java | 12 +-
.../lucene/analysis/bg/BulgarianStemmer.java | 86 +-
.../apache/lucene/analysis/bg/package-info.java | 4 +-
.../apache/lucene/analysis/bn/BengaliAnalyzer.java | 65 +-
.../analysis/bn/BengaliNormalizationFilter.java | 25 +-
.../bn/BengaliNormalizationFilterFactory.java | 14 +-
.../lucene/analysis/bn/BengaliNormalizer.java | 72 +-
.../lucene/analysis/bn/BengaliStemFilter.java | 15 +-
.../analysis/bn/BengaliStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/bn/BengaliStemmer.java | 255 +-
.../apache/lucene/analysis/bn/package-info.java | 4 +-
.../analysis/boost/DelimitedBoostTokenFilter.java | 14 +-
.../boost/DelimitedBoostTokenFilterFactory.java | 16 +-
.../apache/lucene/analysis/boost/package-info.java | 4 +-
.../lucene/analysis/br/BrazilianAnalyzer.java | 72 +-
.../lucene/analysis/br/BrazilianStemFilter.java | 31 +-
.../analysis/br/BrazilianStemFilterFactory.java | 14 +-
.../lucene/analysis/br/BrazilianStemmer.java | 1202 ++++---
.../apache/lucene/analysis/br/package-info.java | 4 +-
.../apache/lucene/analysis/ca/CatalanAnalyzer.java | 66 +-
.../apache/lucene/analysis/ca/package-info.java | 4 +-
.../lucene/analysis/charfilter/BaseCharFilter.java | 49 +-
.../charfilter/HTMLStripCharFilterFactory.java | 11 +-
.../analysis/charfilter/MappingCharFilter.java | 26 +-
.../charfilter/MappingCharFilterFactory.java | 80 +-
.../analysis/charfilter/NormalizeCharMap.java | 46 +-
.../lucene/analysis/charfilter/package-info.java | 61 +-
.../apache/lucene/analysis/cjk/CJKAnalyzer.java | 35 +-
.../lucene/analysis/cjk/CJKBigramFilter.java | 191 +-
.../analysis/cjk/CJKBigramFilterFactory.java | 16 +-
.../lucene/analysis/cjk/CJKWidthCharFilter.java | 67 +-
.../analysis/cjk/CJKWidthCharFilterFactory.java | 8 +-
.../apache/lucene/analysis/cjk/CJKWidthFilter.java | 75 +-
.../lucene/analysis/cjk/CJKWidthFilterFactory.java | 14 +-
.../apache/lucene/analysis/cjk/package-info.java | 29 +-
.../apache/lucene/analysis/ckb/SoraniAnalyzer.java | 57 +-
.../analysis/ckb/SoraniNormalizationFilter.java | 9 +-
.../ckb/SoraniNormalizationFilterFactory.java | 8 +-
.../lucene/analysis/ckb/SoraniNormalizer.java | 28 +-
.../lucene/analysis/ckb/SoraniStemFilter.java | 24 +-
.../analysis/ckb/SoraniStemFilterFactory.java | 8 +-
.../apache/lucene/analysis/ckb/SoraniStemmer.java | 52 +-
.../apache/lucene/analysis/ckb/package-info.java | 4 +-
.../lucene/analysis/classic/ClassicAnalyzer.java | 60 +-
.../lucene/analysis/classic/ClassicFilter.java | 28 +-
.../analysis/classic/ClassicFilterFactory.java | 12 +-
.../lucene/analysis/classic/ClassicTokenizer.java | 99 +-
.../analysis/classic/ClassicTokenizerFactory.java | 7 +-
.../analysis/classic/ClassicTokenizerImpl.java | 706 ++--
.../analysis/classic/ClassicTokenizerImpl.jflex | 2 +-
.../lucene/analysis/classic/package-info.java | 19 +-
.../analysis/commongrams/CommonGramsFilter.java | 80 +-
.../commongrams/CommonGramsFilterFactory.java | 12 +-
.../commongrams/CommonGramsQueryFilter.java | 62 +-
.../commongrams/CommonGramsQueryFilterFactory.java | 10 +-
.../lucene/analysis/commongrams/package-info.java | 4 +-
.../compound/CompoundWordTokenFilterBase.java | 78 +-
.../DictionaryCompoundWordTokenFilter.java | 89 +-
.../DictionaryCompoundWordTokenFilterFactory.java | 24 +-
.../HyphenationCompoundWordTokenFilter.java | 118 +-
.../HyphenationCompoundWordTokenFilterFactory.java | 58 +-
.../analysis/compound/hyphenation/ByteVector.java | 31 +-
.../analysis/compound/hyphenation/CharVector.java | 33 +-
.../analysis/compound/hyphenation/Hyphen.java | 24 +-
.../analysis/compound/hyphenation/Hyphenation.java | 17 +-
.../compound/hyphenation/HyphenationTree.java | 179 +-
.../compound/hyphenation/PatternConsumer.java | 25 +-
.../compound/hyphenation/PatternParser.java | 67 +-
.../analysis/compound/hyphenation/TernaryTree.java | 183 +-
.../compound/hyphenation/package-info.java | 9 +-
.../lucene/analysis/compound/package-info.java | 123 +-
.../lucene/analysis/core/DecimalDigitFilter.java | 16 +-
.../analysis/core/DecimalDigitFilterFactory.java | 12 +-
.../lucene/analysis/core/FlattenGraphFilter.java | 143 +-
.../analysis/core/FlattenGraphFilterFactory.java | 11 +-
.../lucene/analysis/core/KeywordAnalyzer.java | 8 +-
.../lucene/analysis/core/KeywordTokenizer.java | 37 +-
.../analysis/core/KeywordTokenizerFactory.java | 31 +-
.../lucene/analysis/core/LetterTokenizer.java | 47 +-
.../analysis/core/LetterTokenizerFactory.java | 26 +-
.../lucene/analysis/core/LowerCaseFilter.java | 13 +-
.../analysis/core/LowerCaseFilterFactory.java | 11 +-
.../lucene/analysis/core/SimpleAnalyzer.java | 16 +-
.../apache/lucene/analysis/core/StopAnalyzer.java | 41 +-
.../apache/lucene/analysis/core/StopFilter.java | 20 +-
.../lucene/analysis/core/StopFilterFactory.java | 62 +-
.../lucene/analysis/core/TypeTokenFilter.java | 24 +-
.../analysis/core/TypeTokenFilterFactory.java | 18 +-
.../analysis/core/UnicodeWhitespaceAnalyzer.java | 14 +-
.../analysis/core/UnicodeWhitespaceTokenizer.java | 37 +-
.../lucene/analysis/core/UpperCaseFilter.java | 20 +-
.../analysis/core/UpperCaseFilterFactory.java | 21 +-
.../lucene/analysis/core/WhitespaceAnalyzer.java | 12 +-
.../lucene/analysis/core/WhitespaceTokenizer.java | 38 +-
.../analysis/core/WhitespaceTokenizerFactory.java | 28 +-
.../apache/lucene/analysis/core/package-info.java | 4 +-
.../lucene/analysis/custom/CustomAnalyzer.java | 467 +--
.../lucene/analysis/custom/package-info.java | 4 +-
.../apache/lucene/analysis/cz/CzechAnalyzer.java | 61 +-
.../apache/lucene/analysis/cz/CzechStemFilter.java | 24 +-
.../lucene/analysis/cz/CzechStemFilterFactory.java | 12 +-
.../apache/lucene/analysis/cz/CzechStemmer.java | 141 +-
.../apache/lucene/analysis/cz/package-info.java | 4 +-
.../apache/lucene/analysis/da/DanishAnalyzer.java | 57 +-
.../apache/lucene/analysis/da/package-info.java | 4 +-
.../apache/lucene/analysis/de/GermanAnalyzer.java | 93 +-
.../lucene/analysis/de/GermanLightStemFilter.java | 16 +-
.../analysis/de/GermanLightStemFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanLightStemmer.java | 125 +-
.../analysis/de/GermanMinimalStemFilter.java | 16 +-
.../de/GermanMinimalStemFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanMinimalStemmer.java | 99 +-
.../analysis/de/GermanNormalizationFilter.java | 37 +-
.../de/GermanNormalizationFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanStemFilter.java | 93 +-
.../analysis/de/GermanStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/de/GermanStemmer.java | 412 ++-
.../apache/lucene/analysis/de/package-info.java | 4 +-
.../apache/lucene/analysis/el/GreekAnalyzer.java | 66 +-
.../lucene/analysis/el/GreekLowerCaseFilter.java | 66 +-
.../analysis/el/GreekLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/el/GreekStemFilter.java | 33 +-
.../lucene/analysis/el/GreekStemFilterFactory.java | 14 +-
.../apache/lucene/analysis/el/GreekStemmer.java | 1091 +++---
.../apache/lucene/analysis/el/package-info.java | 4 +-
.../analysis/email/UAX29URLEmailAnalyzer.java | 59 +-
.../analysis/email/UAX29URLEmailTokenizer.java | 80 +-
.../email/UAX29URLEmailTokenizerFactory.java | 11 +-
.../apache/lucene/analysis/email/package-info.java | 30 +-
.../apache/lucene/analysis/en/EnglishAnalyzer.java | 61 +-
.../analysis/en/EnglishMinimalStemFilter.java | 16 +-
.../en/EnglishMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/en/EnglishMinimalStemmer.java | 24 +-
.../analysis/en/EnglishPossessiveFilter.java | 20 +-
.../en/EnglishPossessiveFilterFactory.java | 14 +-
.../org/apache/lucene/analysis/en/KStemData1.java | 1412 ++++----
.../org/apache/lucene/analysis/en/KStemData2.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData3.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData4.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData5.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData6.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData7.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData8.java | 1209 ++++---
.../org/apache/lucene/analysis/en/KStemFilter.java | 41 +-
.../lucene/analysis/en/KStemFilterFactory.java | 10 +-
.../org/apache/lucene/analysis/en/KStemmer.java | 1265 ++++---
.../lucene/analysis/en/PorterStemFilter.java | 62 +-
.../analysis/en/PorterStemFilterFactory.java | 11 +-
.../apache/lucene/analysis/en/PorterStemmer.java | 500 +--
.../apache/lucene/analysis/en/package-info.java | 4 +-
.../apache/lucene/analysis/es/SpanishAnalyzer.java | 58 +-
.../lucene/analysis/es/SpanishLightStemFilter.java | 16 +-
.../analysis/es/SpanishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/es/SpanishLightStemmer.java | 92 +-
.../analysis/es/SpanishMinimalStemFilter.java | 13 +-
.../es/SpanishMinimalStemFilterFactory.java | 11 +-
.../lucene/analysis/es/SpanishMinimalStemmer.java | 55 +-
.../apache/lucene/analysis/es/package-info.java | 4 +-
.../lucene/analysis/et/EstonianAnalyzer.java | 155 +-
.../apache/lucene/analysis/et/package-info.java | 4 +-
.../apache/lucene/analysis/eu/BasqueAnalyzer.java | 54 +-
.../apache/lucene/analysis/eu/package-info.java | 4 +-
.../apache/lucene/analysis/fa/PersianAnalyzer.java | 77 +-
.../lucene/analysis/fa/PersianCharFilter.java | 14 +-
.../analysis/fa/PersianCharFilterFactory.java | 7 +-
.../analysis/fa/PersianNormalizationFilter.java | 14 +-
.../fa/PersianNormalizationFilterFactory.java | 14 +-
.../lucene/analysis/fa/PersianNormalizer.java | 53 +-
.../apache/lucene/analysis/fa/package-info.java | 4 +-
.../apache/lucene/analysis/fi/FinnishAnalyzer.java | 57 +-
.../lucene/analysis/fi/FinnishLightStemFilter.java | 16 +-
.../analysis/fi/FinnishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/fi/FinnishLightStemmer.java | 237 +-
.../apache/lucene/analysis/fi/package-info.java | 4 +-
.../apache/lucene/analysis/fr/FrenchAnalyzer.java | 99 +-
.../lucene/analysis/fr/FrenchLightStemFilter.java | 16 +-
.../analysis/fr/FrenchLightStemFilterFactory.java | 12 +-
.../lucene/analysis/fr/FrenchLightStemmer.java | 261 +-
.../analysis/fr/FrenchMinimalStemFilter.java | 16 +-
.../fr/FrenchMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/fr/FrenchMinimalStemmer.java | 66 +-
.../apache/lucene/analysis/fr/package-info.java | 4 +-
.../apache/lucene/analysis/ga/IrishAnalyzer.java | 79 +-
.../lucene/analysis/ga/IrishLowerCaseFilter.java | 22 +-
.../analysis/ga/IrishLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/ga/package-info.java | 4 +-
.../lucene/analysis/gl/GalicianAnalyzer.java | 57 +-
.../analysis/gl/GalicianMinimalStemFilter.java | 16 +-
.../gl/GalicianMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/gl/GalicianMinimalStemmer.java | 17 +-
.../lucene/analysis/gl/GalicianStemFilter.java | 18 +-
.../analysis/gl/GalicianStemFilterFactory.java | 14 +-
.../apache/lucene/analysis/gl/GalicianStemmer.java | 47 +-
.../apache/lucene/analysis/gl/package-info.java | 4 +-
.../apache/lucene/analysis/hi/HindiAnalyzer.java | 65 +-
.../analysis/hi/HindiNormalizationFilter.java | 27 +-
.../hi/HindiNormalizationFilterFactory.java | 16 +-
.../apache/lucene/analysis/hi/HindiNormalizer.java | 271 +-
.../apache/lucene/analysis/hi/HindiStemFilter.java | 12 +-
.../lucene/analysis/hi/HindiStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/hi/HindiStemmer.java | 166 +-
.../apache/lucene/analysis/hi/package-info.java | 4 +-
.../lucene/analysis/hu/HungarianAnalyzer.java | 57 +-
.../analysis/hu/HungarianLightStemFilter.java | 16 +-
.../hu/HungarianLightStemFilterFactory.java | 16 +-
.../lucene/analysis/hu/HungarianLightStemmer.java | 252 +-
.../apache/lucene/analysis/hu/package-info.java | 4 +-
.../lucene/analysis/hunspell/Dictionary.java | 518 +--
.../analysis/hunspell/HunspellStemFilter.java | 96 +-
.../hunspell/HunspellStemFilterFactory.java | 45 +-
.../analysis/hunspell/ISO8859_14Decoder.java | 32 +-
.../apache/lucene/analysis/hunspell/Stemmer.java | 329 +-
.../lucene/analysis/hunspell/package-info.java | 11 +-
.../lucene/analysis/hy/ArmenianAnalyzer.java | 55 +-
.../apache/lucene/analysis/hy/package-info.java | 4 +-
.../lucene/analysis/id/IndonesianAnalyzer.java | 62 +-
.../lucene/analysis/id/IndonesianStemFilter.java | 23 +-
.../analysis/id/IndonesianStemFilterFactory.java | 12 +-
.../lucene/analysis/id/IndonesianStemmer.java | 130 +-
.../apache/lucene/analysis/id/package-info.java | 4 +-
.../analysis/in/IndicNormalizationFilter.java | 9 +-
.../in/IndicNormalizationFilterFactory.java | 16 +-
.../apache/lucene/analysis/in/IndicNormalizer.java | 407 ++-
.../apache/lucene/analysis/in/package-info.java | 4 +-
.../apache/lucene/analysis/it/ItalianAnalyzer.java | 71 +-
.../lucene/analysis/it/ItalianLightStemFilter.java | 16 +-
.../analysis/it/ItalianLightStemFilterFactory.java | 18 +-
.../lucene/analysis/it/ItalianLightStemmer.java | 103 +-
.../apache/lucene/analysis/it/package-info.java | 4 +-
.../lucene/analysis/lt/LithuanianAnalyzer.java | 55 +-
.../apache/lucene/analysis/lt/package-info.java | 4 +-
.../apache/lucene/analysis/lv/LatvianAnalyzer.java | 57 +-
.../lucene/analysis/lv/LatvianStemFilter.java | 16 +-
.../analysis/lv/LatvianStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/lv/LatvianStemmer.java | 155 +-
.../apache/lucene/analysis/lv/package-info.java | 4 +-
.../lucene/analysis/minhash/MinHashFilter.java | 73 +-
.../analysis/minhash/MinHashFilterFactory.java | 17 +-
.../lucene/analysis/minhash/package-info.java | 4 +-
.../analysis/miscellaneous/ASCIIFoldingFilter.java | 134 +-
.../miscellaneous/ASCIIFoldingFilterFactory.java | 15 +-
.../miscellaneous/CapitalizationFilter.java | 73 +-
.../miscellaneous/CapitalizationFilterFactory.java | 52 +-
.../miscellaneous/CodepointCountFilter.java | 21 +-
.../miscellaneous/CodepointCountFilterFactory.java | 10 +-
.../miscellaneous/ConcatenateGraphFilter.java | 150 +-
.../ConcatenateGraphFilterFactory.java | 59 +-
.../miscellaneous/ConcatenatingTokenStream.java | 30 +-
.../miscellaneous/ConditionalTokenFilter.java | 40 +-
.../ConditionalTokenFilterFactory.java | 41 +-
.../miscellaneous/DateRecognizerFilter.java | 11 +-
.../miscellaneous/DateRecognizerFilterFactory.java | 21 +-
.../DelimitedTermFrequencyTokenFilter.java | 23 +-
.../DelimitedTermFrequencyTokenFilterFactory.java | 8 +-
.../miscellaneous/DropIfFlaggedFilter.java | 1 -
.../miscellaneous/DropIfFlaggedFilterFactory.java | 21 +-
.../analysis/miscellaneous/EmptyTokenStream.java | 6 +-
.../analysis/miscellaneous/FingerprintFilter.java | 93 +-
.../miscellaneous/FingerprintFilterFactory.java | 19 +-
.../miscellaneous/FixBrokenOffsetsFilter.java | 3 +-
.../FixBrokenOffsetsFilterFactory.java | 6 +-
.../miscellaneous/HyphenatedWordsFilter.java | 34 +-
.../HyphenatedWordsFilterFactory.java | 13 +-
.../analysis/miscellaneous/KeepWordFilter.java | 16 +-
.../miscellaneous/KeepWordFilterFactory.java | 11 +-
.../miscellaneous/KeywordMarkerFilter.java | 10 +-
.../miscellaneous/KeywordMarkerFilterFactory.java | 21 +-
.../miscellaneous/KeywordRepeatFilter.java | 20 +-
.../miscellaneous/KeywordRepeatFilterFactory.java | 17 +-
.../analysis/miscellaneous/LengthFilter.java | 21 +-
.../miscellaneous/LengthFilterFactory.java | 11 +-
.../miscellaneous/LimitTokenCountAnalyzer.java | 33 +-
.../miscellaneous/LimitTokenCountFilter.java | 38 +-
.../LimitTokenCountFilterFactory.java | 15 +-
.../miscellaneous/LimitTokenOffsetFilter.java | 23 +-
.../LimitTokenOffsetFilterFactory.java | 10 +-
.../miscellaneous/LimitTokenPositionFilter.java | 43 +-
.../LimitTokenPositionFilterFactory.java | 16 +-
.../miscellaneous/PatternKeywordMarkerFilter.java | 27 +-
.../miscellaneous/PerFieldAnalyzerWrapper.java | 57 +-
.../miscellaneous/ProtectedTermFilter.java | 18 +-
.../miscellaneous/ProtectedTermFilterFactory.java | 72 +-
.../miscellaneous/RemoveDuplicatesTokenFilter.java | 23 +-
.../RemoveDuplicatesTokenFilterFactory.java | 11 +-
.../miscellaneous/ScandinavianFoldingFilter.java | 81 +-
.../ScandinavianFoldingFilterFactory.java | 8 +-
.../ScandinavianNormalizationFilter.java | 55 +-
.../ScandinavianNormalizationFilterFactory.java | 6 +-
.../miscellaneous/SetKeywordMarkerFilter.java | 23 +-
.../miscellaneous/StemmerOverrideFilter.java | 95 +-
.../StemmerOverrideFilterFactory.java | 11 +-
.../lucene/analysis/miscellaneous/TrimFilter.java | 23 +-
.../analysis/miscellaneous/TrimFilterFactory.java | 12 +-
.../miscellaneous/TruncateTokenFilter.java | 16 +-
.../miscellaneous/TruncateTokenFilterFactory.java | 14 +-
.../miscellaneous/TypeAsSynonymFilter.java | 28 +-
.../miscellaneous/TypeAsSynonymFilterFactory.java | 24 +-
.../miscellaneous/WordDelimiterFilter.java | 267 +-
.../miscellaneous/WordDelimiterFilterFactory.java | 134 +-
.../miscellaneous/WordDelimiterGraphFilter.java | 325 +-
.../WordDelimiterGraphFilterFactory.java | 135 +-
.../miscellaneous/WordDelimiterIterator.java | 184 +-
.../analysis/miscellaneous/package-info.java | 6 +-
.../analysis/ngram/EdgeNGramFilterFactory.java | 8 +-
.../analysis/ngram/EdgeNGramTokenFilter.java | 42 +-
.../lucene/analysis/ngram/EdgeNGramTokenizer.java | 12 +-
.../analysis/ngram/EdgeNGramTokenizerFactory.java | 7 +-
.../lucene/analysis/ngram/NGramFilterFactory.java | 8 +-
.../lucene/analysis/ngram/NGramTokenFilter.java | 71 +-
.../lucene/analysis/ngram/NGramTokenizer.java | 64 +-
.../analysis/ngram/NGramTokenizerFactory.java | 14 +-
.../apache/lucene/analysis/ngram/package-info.java | 4 +-
.../apache/lucene/analysis/nl/DutchAnalyzer.java | 92 +-
.../apache/lucene/analysis/nl/package-info.java | 4 +-
.../lucene/analysis/no/NorwegianAnalyzer.java | 58 +-
.../analysis/no/NorwegianLightStemFilter.java | 33 +-
.../no/NorwegianLightStemFilterFactory.java | 20 +-
.../lucene/analysis/no/NorwegianLightStemmer.java | 224 +-
.../analysis/no/NorwegianMinimalStemFilter.java | 31 +-
.../no/NorwegianMinimalStemFilterFactory.java | 20 +-
.../analysis/no/NorwegianMinimalStemmer.java | 95 +-
.../apache/lucene/analysis/no/package-info.java | 4 +-
.../analysis/path/PathHierarchyTokenizer.java | 76 +-
.../path/PathHierarchyTokenizerFactory.java | 48 +-
.../path/ReversePathHierarchyTokenizer.java | 45 +-
.../apache/lucene/analysis/path/package-info.java | 4 +-
.../pattern/PatternCaptureGroupFilterFactory.java | 18 +-
.../pattern/PatternCaptureGroupTokenFilter.java | 86 +-
.../analysis/pattern/PatternReplaceCharFilter.java | 49 +-
.../pattern/PatternReplaceCharFilterFactory.java | 9 +-
.../analysis/pattern/PatternReplaceFilter.java | 36 +-
.../pattern/PatternReplaceFilterFactory.java | 12 +-
.../lucene/analysis/pattern/PatternTokenizer.java | 61 +-
.../analysis/pattern/PatternTokenizerFactory.java | 52 +-
.../pattern/SimplePatternSplitTokenizer.java | 43 +-
.../SimplePatternSplitTokenizerFactory.java | 45 +-
.../analysis/pattern/SimplePatternTokenizer.java | 55 +-
.../pattern/SimplePatternTokenizerFactory.java | 40 +-
.../lucene/analysis/pattern/package-info.java | 4 +-
.../lucene/analysis/payloads/AbstractEncoder.java | 8 +-
.../payloads/DelimitedPayloadTokenFilter.java | 24 +-
.../DelimitedPayloadTokenFilterFactory.java | 18 +-
.../lucene/analysis/payloads/FloatEncoder.java | 8 +-
.../lucene/analysis/payloads/IdentityEncoder.java | 14 +-
.../lucene/analysis/payloads/IntegerEncoder.java | 13 +-
.../payloads/NumericPayloadTokenFilter.java | 13 +-
.../payloads/NumericPayloadTokenFilterFactory.java | 13 +-
.../lucene/analysis/payloads/PayloadEncoder.java | 13 +-
.../lucene/analysis/payloads/PayloadHelper.java | 36 +-
.../payloads/TokenOffsetPayloadTokenFilter.java | 15 +-
.../TokenOffsetPayloadTokenFilterFactory.java | 14 +-
.../payloads/TypeAsPayloadTokenFilter.java | 11 +-
.../payloads/TypeAsPayloadTokenFilterFactory.java | 14 +-
.../lucene/analysis/payloads/package-info.java | 4 +-
.../lucene/analysis/pt/PortugueseAnalyzer.java | 58 +-
.../analysis/pt/PortugueseLightStemFilter.java | 16 +-
.../pt/PortugueseLightStemFilterFactory.java | 14 +-
.../lucene/analysis/pt/PortugueseLightStemmer.java | 162 +-
.../analysis/pt/PortugueseMinimalStemFilter.java | 16 +-
.../pt/PortugueseMinimalStemFilterFactory.java | 14 +-
.../analysis/pt/PortugueseMinimalStemmer.java | 22 +-
.../lucene/analysis/pt/PortugueseStemFilter.java | 18 +-
.../analysis/pt/PortugueseStemFilterFactory.java | 16 +-
.../lucene/analysis/pt/PortugueseStemmer.java | 65 +-
.../apache/lucene/analysis/pt/RSLPStemmerBase.java | 262 +-
.../apache/lucene/analysis/pt/package-info.java | 4 +-
.../analysis/query/QueryAutoStopWordAnalyzer.java | 90 +-
.../apache/lucene/analysis/query/package-info.java | 4 +-
.../analysis/reverse/ReverseStringFilter.java | 106 +-
.../reverse/ReverseStringFilterFactory.java | 12 +-
.../lucene/analysis/reverse/package-info.java | 4 +-
.../lucene/analysis/ro/RomanianAnalyzer.java | 61 +-
.../apache/lucene/analysis/ro/package-info.java | 4 +-
.../apache/lucene/analysis/ru/RussianAnalyzer.java | 159 +-
.../lucene/analysis/ru/RussianLightStemFilter.java | 16 +-
.../analysis/ru/RussianLightStemFilterFactory.java | 14 +-
.../lucene/analysis/ru/RussianLightStemmer.java | 168 +-
.../apache/lucene/analysis/ru/package-info.java | 4 +-
.../analysis/shingle/FixedShingleFilter.java | 45 +-
.../shingle/FixedShingleFilterFactory.java | 6 +-
.../analysis/shingle/ShingleAnalyzerWrapper.java | 52 +-
.../lucene/analysis/shingle/ShingleFilter.java | 295 +-
.../analysis/shingle/ShingleFilterFactory.java | 24 +-
.../lucene/analysis/shingle/package-info.java | 4 +-
.../lucene/analysis/sinks/TeeSinkTokenFilter.java | 44 +-
.../apache/lucene/analysis/sinks/package-info.java | 4 +-
.../lucene/analysis/snowball/SnowballFilter.java | 42 +-
.../snowball/SnowballPorterFilterFactory.java | 20 +-
.../lucene/analysis/snowball/package-info.java | 53 +-
.../apache/lucene/analysis/sr/SerbianAnalyzer.java | 49 +-
.../analysis/sr/SerbianNormalizationFilter.java | 243 +-
.../sr/SerbianNormalizationFilterFactory.java | 16 +-
.../sr/SerbianNormalizationRegularFilter.java | 228 +-
.../apache/lucene/analysis/sr/package-info.java | 4 +-
.../apache/lucene/analysis/sv/SwedishAnalyzer.java | 57 +-
.../lucene/analysis/sv/SwedishLightStemFilter.java | 16 +-
.../analysis/sv/SwedishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/sv/SwedishLightStemmer.java | 112 +-
.../apache/lucene/analysis/sv/package-info.java | 4 +-
.../lucene/analysis/synonym/SolrSynonymParser.java | 79 +-
.../lucene/analysis/synonym/SynonymFilter.java | 193 +-
.../analysis/synonym/SynonymFilterFactory.java | 114 +-
.../analysis/synonym/SynonymGraphFilter.java | 185 +-
.../synonym/SynonymGraphFilterFactory.java | 110 +-
.../apache/lucene/analysis/synonym/SynonymMap.java | 128 +-
.../analysis/synonym/WordnetSynonymParser.java | 28 +-
.../lucene/analysis/synonym/package-info.java | 4 +-
.../apache/lucene/analysis/th/ThaiAnalyzer.java | 44 +-
.../apache/lucene/analysis/th/ThaiTokenizer.java | 45 +-
.../lucene/analysis/th/ThaiTokenizerFactory.java | 12 +-
.../apache/lucene/analysis/th/package-info.java | 4 +-
.../lucene/analysis/tr/ApostropheFilter.java | 20 +-
.../analysis/tr/ApostropheFilterFactory.java | 8 +-
.../apache/lucene/analysis/tr/TurkishAnalyzer.java | 54 +-
.../lucene/analysis/tr/TurkishLowerCaseFilter.java | 74 +-
.../analysis/tr/TurkishLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/tr/package-info.java | 4 +-
.../lucene/analysis/util/CharArrayIterator.java | 45 +-
.../apache/lucene/analysis/util/CharTokenizer.java | 127 +-
.../apache/lucene/analysis/util/ElisionFilter.java | 15 +-
.../lucene/analysis/util/ElisionFilterFactory.java | 12 +-
.../analysis/util/FilesystemResourceLoader.java | 55 +-
.../lucene/analysis/util/OpenStringBuilder.java | 55 +-
.../lucene/analysis/util/RollingCharBuffer.java | 46 +-
.../analysis/util/SegmentingTokenizerBase.java | 97 +-
.../apache/lucene/analysis/util/StemmerUtil.java | 44 +-
.../apache/lucene/analysis/util/UnicodeProps.java | 24 +-
.../apache/lucene/analysis/util/package-info.java | 4 +-
.../analysis/wikipedia/WikipediaTokenizer.java | 218 +-
.../wikipedia/WikipediaTokenizerFactory.java | 6 +-
.../analysis/wikipedia/WikipediaTokenizerImpl.java | 1444 ++++----
.../lucene/analysis/wikipedia/package-info.java | 4 +-
.../lucene/analysis/bg/TestBulgarianAnalyzer.java | 37 +-
.../bg/TestBulgarianStemFilterFactory.java | 25 +-
.../lucene/analysis/bg/TestBulgarianStemmer.java | 103 +-
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 12 +-
.../lucene/analysis/bn/TestBengaliFilters.java | 59 +-
.../lucene/analysis/bn/TestBengaliNormalizer.java | 40 +-
.../lucene/analysis/bn/TestBengaliStemmer.java | 31 +-
.../boost/DelimitedBoostTokenFilterTest.java | 22 +-
.../lucene/analysis/br/TestBrazilianAnalyzer.java | 211 +-
.../br/TestBrazilianStemFilterFactory.java | 25 +-
.../lucene/analysis/ca/TestCatalanAnalyzer.java | 21 +-
.../charfilter/HTMLStripCharFilterTest.java | 621 ++--
.../charfilter/TestHTMLStripCharFilterFactory.java | 83 +-
.../analysis/charfilter/TestMappingCharFilter.java | 286 +-
.../charfilter/TestMappingCharFilterFactory.java | 52 +-
.../lucene/analysis/cjk/TestCJKAnalyzer.java | 476 +--
.../lucene/analysis/cjk/TestCJKBigramFilter.java | 287 +-
.../analysis/cjk/TestCJKBigramFilterFactory.java | 51 +-
.../analysis/cjk/TestCJKWidthCharFilter.java | 78 +-
.../cjk/TestCJKWidthCharFilterFactory.java | 20 +-
.../lucene/analysis/cjk/TestCJKWidthFilter.java | 73 +-
.../analysis/cjk/TestCJKWidthFilterFactory.java | 21 +-
.../lucene/analysis/ckb/TestSoraniAnalyzer.java | 32 +-
.../ckb/TestSoraniNormalizationFilter.java | 58 +-
.../ckb/TestSoraniNormalizationFilterFactory.java | 23 +-
.../lucene/analysis/ckb/TestSoraniStemFilter.java | 68 +-
.../analysis/ckb/TestSoraniStemFilterFactory.java | 25 +-
.../analysis/classic/TestClassicAnalyzer.java | 201 +-
.../analysis/classic/TestClassicFactories.java | 54 +-
.../commongrams/CommonGramsFilterTest.java | 390 +--
.../commongrams/TestCommonGramsFilterFactory.java | 81 +-
.../TestCommonGramsQueryFilterFactory.java | 90 +-
.../compound/TestCompoundWordTokenFilter.java | 557 ++--
...stDictionaryCompoundWordTokenFilterFactory.java | 41 +-
...tHyphenationCompoundWordTokenFilterFactory.java | 80 +-
.../core/TestAllAnalyzersHaveFactories.java | 148 +-
.../apache/lucene/analysis/core/TestAnalyzers.java | 162 +-
.../lucene/analysis/core/TestBugInSomething.java | 323 +-
.../lucene/analysis/core/TestCoreFactories.java | 55 +-
.../analysis/core/TestDecimalDigitFilter.java | 121 +-
.../core/TestDecimalDigitFilterFactory.java | 29 +-
.../lucene/analysis/core/TestDuelingAnalyzers.java | 181 +-
.../apache/lucene/analysis/core/TestFactories.java | 73 +-
.../analysis/core/TestFlattenGraphFilter.java | 372 ++-
.../lucene/analysis/core/TestKeywordAnalyzer.java | 26 +-
.../lucene/analysis/core/TestKeywordTokenizer.java | 37 +-
.../lucene/analysis/core/TestRandomChains.java | 776 +++--
.../lucene/analysis/core/TestStopAnalyzer.java | 27 +-
.../analysis/core/TestStopFilterFactory.java | 61 +-
.../lucene/analysis/core/TestTypeTokenFilter.java | 28 +-
.../analysis/core/TestTypeTokenFilterFactory.java | 52 +-
.../core/TestUnicodeWhitespaceTokenizer.java | 54 +-
.../analysis/core/TestWhitespaceAnalyzer.java | 22 +-
.../lucene/analysis/custom/TestCustomAnalyzer.java | 552 ++--
.../lucene/analysis/cz/TestCzechAnalyzer.java | 27 +-
.../analysis/cz/TestCzechStemFilterFactory.java | 27 +-
.../lucene/analysis/cz/TestCzechStemmer.java | 410 ++-
.../lucene/analysis/da/TestDanishAnalyzer.java | 16 +-
.../lucene/analysis/de/TestGermanAnalyzer.java | 25 +-
.../analysis/de/TestGermanLightStemFilter.java | 69 +-
.../de/TestGermanLightStemFilterFactory.java | 21 +-
.../analysis/de/TestGermanMinimalStemFilter.java | 71 +-
.../de/TestGermanMinimalStemFilterFactory.java | 23 +-
.../analysis/de/TestGermanNormalizationFilter.java | 62 +-
.../de/TestGermanNormalizationFilterFactory.java | 23 +-
.../lucene/analysis/de/TestGermanStemFilter.java | 72 +-
.../analysis/de/TestGermanStemFilterFactory.java | 27 +-
.../lucene/analysis/el/GreekAnalyzerTest.java | 45 +-
.../el/TestGreekLowerCaseFilterFactory.java | 25 +-
.../analysis/el/TestGreekStemFilterFactory.java | 21 +-
.../lucene/analysis/el/TestGreekStemmer.java | 186 +-
.../analysis/email/TestUAX29URLEmailAnalyzer.java | 657 ++--
.../analysis/email/TestUAX29URLEmailTokenizer.java | 2426 +++++++++++---
.../email/TestUAX29URLEmailTokenizerFactory.java | 201 +-
.../lucene/analysis/en/TestEnglishAnalyzer.java | 16 +-
.../analysis/en/TestEnglishMinimalStemFilter.java | 46 +-
.../en/TestEnglishMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/en/TestKStemFilterFactory.java | 21 +-
.../apache/lucene/analysis/en/TestKStemmer.java | 113 +-
.../lucene/analysis/en/TestPorterStemFilter.java | 58 +-
.../analysis/en/TestPorterStemFilterFactory.java | 25 +-
.../lucene/analysis/es/TestSpanishAnalyzer.java | 16 +-
.../analysis/es/TestSpanishLightStemFilter.java | 48 +-
.../es/TestSpanishLightStemFilterFactory.java | 23 +-
.../analysis/es/TestSpanishMinimalStemFilter.java | 38 +-
.../es/TestSpanishMinimalStemFilterFactory.java | 20 +-
.../lucene/analysis/et/TestEstonianAnalyzer.java | 69 +-
.../lucene/analysis/eu/TestBasqueAnalyzer.java | 18 +-
.../lucene/analysis/fa/TestPersianAnalyzer.java | 200 +-
.../lucene/analysis/fa/TestPersianCharFilter.java | 34 +-
.../fa/TestPersianNormalizationFilter.java | 30 +-
.../fa/TestPersianNormalizationFilterFactory.java | 25 +-
.../lucene/analysis/fi/TestFinnishAnalyzer.java | 16 +-
.../analysis/fi/TestFinnishLightStemFilter.java | 69 +-
.../fi/TestFinnishLightStemFilterFactory.java | 23 +-
.../lucene/analysis/fr/TestFrenchAnalyzer.java | 142 +-
.../analysis/fr/TestFrenchLightStemFilter.java | 147 +-
.../fr/TestFrenchLightStemFilterFactory.java | 21 +-
.../analysis/fr/TestFrenchMinimalStemFilter.java | 73 +-
.../fr/TestFrenchMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/ga/TestIrishAnalyzer.java | 29 +-
.../analysis/ga/TestIrishLowerCaseFilter.java | 35 +-
.../ga/TestIrishLowerCaseFilterFactory.java | 21 +-
.../lucene/analysis/gl/TestGalicianAnalyzer.java | 16 +-
.../analysis/gl/TestGalicianMinimalStemFilter.java | 69 +-
.../gl/TestGalicianMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/gl/TestGalicianStemFilter.java | 44 +-
.../analysis/gl/TestGalicianStemFilterFactory.java | 21 +-
.../lucene/analysis/hi/TestHindiAnalyzer.java | 25 +-
.../lucene/analysis/hi/TestHindiFilters.java | 59 +-
.../lucene/analysis/hi/TestHindiNormalizer.java | 31 +-
.../lucene/analysis/hi/TestHindiStemmer.java | 58 +-
.../lucene/analysis/hu/TestHungarianAnalyzer.java | 16 +-
.../analysis/hu/TestHungarianLightStemFilter.java | 67 +-
.../hu/TestHungarianLightStemFilterFactory.java | 21 +-
.../lucene/analysis/hunspell/StemmerTestBase.java | 34 +-
.../lucene/analysis/hunspell/Test64kAffixes.java | 18 +-
.../analysis/hunspell/TestAllDictionaries.java | 308 +-
.../analysis/hunspell/TestAllDictionaries2.java | 424 ++-
.../analysis/hunspell/TestAlternateCasing.java | 35 +-
.../analysis/hunspell/TestCaseInsensitive.java | 5 +-
.../analysis/hunspell/TestCaseSensitive.java | 41 +-
.../lucene/analysis/hunspell/TestCircumfix.java | 5 +-
.../analysis/hunspell/TestComplexPrefix.java | 3 +-
.../lucene/analysis/hunspell/TestCondition.java | 3 +-
.../lucene/analysis/hunspell/TestCondition2.java | 5 +-
.../apache/lucene/analysis/hunspell/TestConv.java | 5 +-
.../lucene/analysis/hunspell/TestDependencies.java | 5 +-
.../lucene/analysis/hunspell/TestDictionary.java | 148 +-
.../lucene/analysis/hunspell/TestDoubleEscape.java | 3 +-
.../lucene/analysis/hunspell/TestEscaped.java | 3 +-
.../lucene/analysis/hunspell/TestFlagLong.java | 3 +-
.../lucene/analysis/hunspell/TestFlagNum.java | 3 +-
.../lucene/analysis/hunspell/TestFullStrip.java | 3 +-
.../lucene/analysis/hunspell/TestHomonyms.java | 5 +-
.../analysis/hunspell/TestHunspellStemFilter.java | 82 +-
.../hunspell/TestHunspellStemFilterFactory.java | 30 +-
.../lucene/analysis/hunspell/TestIgnore.java | 5 +-
.../lucene/analysis/hunspell/TestKeepCase.java | 23 +-
.../apache/lucene/analysis/hunspell/TestMorph.java | 5 +-
.../lucene/analysis/hunspell/TestMorphAlias.java | 9 +-
.../lucene/analysis/hunspell/TestMorphData.java | 9 +-
.../lucene/analysis/hunspell/TestNeedAffix.java | 15 +-
.../analysis/hunspell/TestOnlyInCompound.java | 9 +-
.../analysis/hunspell/TestOptionalCondition.java | 3 +-
.../lucene/analysis/hunspell/TestSpaces.java | 5 +-
.../lucene/analysis/hunspell/TestStemmer.java | 5 +-
.../hunspell/TestStrangeOvergeneration.java | 3 +-
.../lucene/analysis/hunspell/TestTwoFold.java | 5 +-
.../lucene/analysis/hunspell/TestTwoSuffixes.java | 5 +-
.../lucene/analysis/hunspell/TestZeroAffix.java | 3 +-
.../lucene/analysis/hunspell/TestZeroAffix2.java | 5 +-
.../lucene/analysis/hy/TestArmenianAnalyzer.java | 18 +-
.../lucene/analysis/id/TestIndonesianAnalyzer.java | 16 +-
.../id/TestIndonesianStemFilterFactory.java | 37 +-
.../lucene/analysis/id/TestIndonesianStemmer.java | 75 +-
.../lucene/analysis/in/TestIndicNormalizer.java | 34 +-
.../lucene/analysis/it/TestItalianAnalyzer.java | 22 +-
.../analysis/it/TestItalianLightStemFilter.java | 48 +-
.../it/TestItalianLightStemFilterFactory.java | 21 +-
.../lucene/analysis/lt/TestLithuanianAnalyzer.java | 18 +-
.../lucene/analysis/lt/TestLithuanianStemming.java | 773 +++--
.../lucene/analysis/lv/TestLatvianAnalyzer.java | 16 +-
.../analysis/lv/TestLatvianStemFilterFactory.java | 23 +-
.../lucene/analysis/lv/TestLatvianStemmer.java | 330 +-
.../lucene/analysis/minhash/MinHashFilterTest.java | 133 +-
.../DateRecognizerFilterFactoryTest.java | 20 +-
.../miscellaneous/DateRecognizerFilterTest.java | 9 +-
.../DelimitedTermFrequencyTokenFilterTest.java | 11 +-
.../miscellaneous/TestASCIIFoldingFilter.java | 3475 ++++++++++----------
.../TestAsciiFoldingFilterFactory.java | 18 +-
.../miscellaneous/TestCapitalizationFilter.java | 366 ++-
.../TestCapitalizationFilterFactory.java | 425 ++-
.../miscellaneous/TestCodepointCountFilter.java | 37 +-
.../TestCodepointCountFilterFactory.java | 55 +-
.../miscellaneous/TestConcatenateGraphFilter.java | 72 +-
.../TestConcatenateGraphFilterFactory.java | 64 +-
.../TestConcatenatingTokenStream.java | 73 +-
.../miscellaneous/TestConditionalTokenFilter.java | 276 +-
.../miscellaneous/TestDropIfFlaggedFilter.java | 39 +-
.../TestDropIfFlaggedFilterFactory.java | 18 +-
.../miscellaneous/TestEmptyTokenStream.java | 11 +-
.../miscellaneous/TestFingerprintFilter.java | 26 +-
.../TestFingerprintFilterFactory.java | 42 +-
.../miscellaneous/TestFixBrokenOffsetsFilter.java | 7 +-
.../miscellaneous/TestHyphenatedWordsFilter.java | 71 +-
.../miscellaneous/TestKeepFilterFactory.java | 26 +-
.../analysis/miscellaneous/TestKeepWordFilter.java | 49 +-
.../miscellaneous/TestKeywordMarkerFilter.java | 122 +-
.../TestKeywordMarkerFilterFactory.java | 108 +-
.../miscellaneous/TestKeywordRepeatFilter.java | 33 +-
.../analysis/miscellaneous/TestLengthFilter.java | 31 +-
.../miscellaneous/TestLengthFilterFactory.java | 58 +-
.../miscellaneous/TestLimitTokenCountAnalyzer.java | 55 +-
.../miscellaneous/TestLimitTokenCountFilter.java | 5 +-
.../TestLimitTokenCountFilterFactory.java | 56 +-
.../miscellaneous/TestLimitTokenOffsetFilter.java | 7 +-
.../TestLimitTokenOffsetFilterFactory.java | 54 +-
.../TestLimitTokenPositionFilter.java | 91 +-
.../TestLimitTokenPositionFilterFactory.java | 75 +-
.../miscellaneous/TestMiscellaneousFactories.java | 29 +-
.../miscellaneous/TestPerFieldAnalyzerWrapper.java | 108 +-
.../miscellaneous/TestProtectedTermFilter.java | 17 +-
.../TestProtectedTermFilterFactory.java | 207 +-
.../TestRemoveDuplicatesTokenFilter.java | 191 +-
.../TestRemoveDuplicatesTokenFilterFactory.java | 36 +-
.../TestScandinavianFoldingFilter.java | 45 +-
.../TestScandinavianFoldingFilterFactory.java | 19 +-
.../TestScandinavianNormalizationFilter.java | 46 +-
...TestScandinavianNormalizationFilterFactory.java | 16 +-
.../miscellaneous/TestStemmerOverrideFilter.java | 62 +-
.../TestStemmerOverrideFilterFactory.java | 52 +-
.../analysis/miscellaneous/TestTrimFilter.java | 52 +-
.../miscellaneous/TestTrimFilterFactory.java | 21 +-
.../miscellaneous/TestTruncateTokenFilter.java | 8 +-
.../TestTruncateTokenFilterFactory.java | 68 +-
.../miscellaneous/TestTypeAsSynonymFilter.java | 87 +-
.../TestTypeAsSynonymFilterFactory.java | 37 +-
.../miscellaneous/TestWordDelimiterFilter.java | 651 ++--
.../TestWordDelimiterGraphFilter.java | 1062 +++---
.../analysis/ngram/EdgeNGramTokenFilterTest.java | 160 +-
.../analysis/ngram/EdgeNGramTokenizerTest.java | 116 +-
.../analysis/ngram/NGramTokenFilterTest.java | 224 +-
.../lucene/analysis/ngram/NGramTokenizerTest.java | 131 +-
.../lucene/analysis/ngram/TestNGramFilters.java | 180 +-
.../lucene/analysis/nl/TestDutchAnalyzer.java | 210 +-
.../lucene/analysis/no/TestNorwegianAnalyzer.java | 16 +-
.../analysis/no/TestNorwegianLightStemFilter.java | 90 +-
.../no/TestNorwegianLightStemFilterFactory.java | 23 +-
.../no/TestNorwegianMinimalStemFilter.java | 92 +-
.../no/TestNorwegianMinimalStemFilterFactory.java | 35 +-
.../analysis/path/TestPathHierarchyTokenizer.java | 264 +-
.../path/TestReversePathHierarchyTokenizer.java | 219 +-
.../TestPatternCaptureGroupTokenFilter.java | 554 ++--
.../pattern/TestPatternReplaceCharFilter.java | 242 +-
.../TestPatternReplaceCharFilterFactory.java | 84 +-
.../analysis/pattern/TestPatternReplaceFilter.java | 115 +-
.../pattern/TestPatternReplaceFilterFactory.java | 34 +-
.../analysis/pattern/TestPatternTokenizer.java | 125 +-
.../pattern/TestPatternTokenizerFactory.java | 23 +-
.../pattern/TestSimplePatternSplitTokenizer.java | 135 +-
.../pattern/TestSimplePatternTokenizer.java | 97 +-
.../payloads/DelimitedPayloadTokenFilterTest.java | 52 +-
.../payloads/NumericPayloadTokenFilterTest.java | 33 +-
.../TestDelimitedPayloadTokenFilterFactory.java | 27 +-
.../TokenOffsetPayloadTokenFilterTest.java | 16 +-
.../payloads/TypeAsPayloadTokenFilterTest.java | 20 +-
.../lucene/analysis/pt/TestPortugueseAnalyzer.java | 16 +-
.../analysis/pt/TestPortugueseLightStemFilter.java | 166 +-
.../pt/TestPortugueseLightStemFilterFactory.java | 23 +-
.../pt/TestPortugueseMinimalStemFilter.java | 120 +-
.../pt/TestPortugueseMinimalStemFilterFactory.java | 23 +-
.../analysis/pt/TestPortugueseStemFilter.java | 120 +-
.../pt/TestPortugueseStemFilterFactory.java | 21 +-
.../query/QueryAutoStopWordAnalyzerTest.java | 65 +-
.../analysis/reverse/TestReverseStringFilter.java | 67 +-
.../reverse/TestReverseStringFilterFactory.java | 25 +-
.../lucene/analysis/ro/TestRomanianAnalyzer.java | 16 +-
.../lucene/analysis/ru/TestRussianAnalyzer.java | 41 +-
.../analysis/ru/TestRussianLightStemFilter.java | 69 +-
.../ru/TestRussianLightStemFilterFactory.java | 21 +-
.../analysis/shingle/FixedShingleFilterTest.java | 362 +-
.../shingle/ShingleAnalyzerWrapperTest.java | 551 ++--
.../lucene/analysis/shingle/ShingleFilterTest.java | 2240 +++++++------
.../analysis/shingle/TestShingleFilterFactory.java | 238 +-
.../analysis/sinks/TestTeeSinkTokenFilter.java | 74 +-
.../lucene/analysis/snowball/TestSnowball.java | 86 +-
.../snowball/TestSnowballPorterFilterFactory.java | 43 +-
.../analysis/snowball/TestSnowballVocab.java | 37 +-
.../lucene/analysis/sr/TestSerbianAnalyzer.java | 16 +-
.../sr/TestSerbianNormalizationFilter.java | 57 +-
.../sr/TestSerbianNormalizationFilterFactory.java | 28 +-
.../sr/TestSerbianNormalizationRegularFilter.java | 58 +-
.../lucene/analysis/sv/TestSwedishAnalyzer.java | 16 +-
.../analysis/sv/TestSwedishLightStemFilter.java | 69 +-
.../sv/TestSwedishLightStemFilterFactory.java | 21 +-
.../synonym/BaseSynonymParserTestCase.java | 61 +-
.../analysis/synonym/TestMultiWordSynonyms.java | 27 +-
.../analysis/synonym/TestSolrSynonymParser.java | 232 +-
.../analysis/synonym/TestSynonymFilterFactory.java | 148 +-
.../analysis/synonym/TestSynonymGraphFilter.java | 1330 ++++----
.../analysis/synonym/TestSynonymMapFilter.java | 628 ++--
.../analysis/synonym/TestWordnetSynonymParser.java | 70 +-
.../lucene/analysis/th/TestThaiAnalyzer.java | 110 +-
.../analysis/th/TestThaiTokenizerFactory.java | 33 +-
.../lucene/analysis/tr/TestApostropheFilter.java | 4 +-
.../analysis/tr/TestApostropheFilterFactory.java | 32 +-
.../lucene/analysis/tr/TestTurkishAnalyzer.java | 13 +-
.../analysis/tr/TestTurkishLowerCaseFilter.java | 72 +-
.../tr/TestTurkishLowerCaseFilterFactory.java | 27 +-
.../analysis/util/StringMockResourceLoader.java | 6 +-
.../analysis/util/TestCharArrayIterator.java | 46 +-
.../lucene/analysis/util/TestCharTokenizers.java | 171 +-
.../apache/lucene/analysis/util/TestElision.java | 25 +-
.../analysis/util/TestElisionFilterFactory.java | 51 +-
.../util/TestFilesystemResourceLoader.java | 61 +-
.../analysis/util/TestRollingCharBuffer.java | 17 +-
.../analysis/util/TestSegmentingTokenizerBase.java | 177 +-
.../wikipedia/TestWikipediaTokenizerFactory.java | 173 +-
.../analysis/wikipedia/WikipediaTokenizerTest.java | 521 ++-
.../analysis/standard/GenerateJflexTLDMacros.java | 177 +-
.../src/java/org/apache/lucene/LucenePackage.java | 2 +-
.../apache/lucene/analysis/LowerCaseFilter.java | 4 +-
.../tokenattributes/PackedTokenAttributeImpl.java | 4 +-
.../org/apache/lucene/search/PhrasePositions.java | 4 +-
.../org/apache/lucene/util/AttributeSource.java | 4 +-
.../java/org/apache/lucene/util/UnicodeUtil.java | 8 +-
.../apache/lucene/util/automaton/StatePair.java | 4 +-
.../apache/lucene/queries/CommonTermsQuery.java | 249 +-
.../queries/function/FunctionMatchQuery.java | 51 +-
.../lucene/queries/function/FunctionQuery.java | 42 +-
.../queries/function/FunctionRangeQuery.java | 79 +-
.../queries/function/FunctionScoreQuery.java | 140 +-
.../lucene/queries/function/FunctionValues.java | 154 +-
.../queries/function/IndexReaderFunctions.java | 27 +-
.../lucene/queries/function/ValueSource.java | 88 +-
.../lucene/queries/function/ValueSourceScorer.java | 55 +-
.../queries/function/docvalues/BoolDocValues.java | 11 +-
.../function/docvalues/DocTermsIndexDocValues.java | 37 +-
.../function/docvalues/DoubleDocValues.java | 40 +-
.../queries/function/docvalues/FloatDocValues.java | 11 +-
.../queries/function/docvalues/IntDocValues.java | 27 +-
.../queries/function/docvalues/LongDocValues.java | 29 +-
.../queries/function/docvalues/StrDocValues.java | 1 -
.../queries/function/docvalues/package-info.java | 4 +-
.../lucene/queries/function/package-info.java | 4 +-
.../queries/function/valuesource/BoolFunction.java | 4 +-
.../function/valuesource/BytesRefFieldSource.java | 16 +-
.../valuesource/ComparisonBoolFunction.java | 22 +-
.../function/valuesource/ConstNumberSource.java | 13 +-
.../function/valuesource/ConstValueSource.java | 30 +-
.../queries/function/valuesource/DefFunction.java | 23 +-
.../function/valuesource/DivFloatFunction.java | 12 +-
.../function/valuesource/DocFreqValueSource.java | 38 +-
.../valuesource/DoubleConstValueSource.java | 20 +-
.../function/valuesource/DoubleFieldSource.java | 20 +-
.../function/valuesource/DualFloatFunction.java | 44 +-
.../function/valuesource/EnumFieldSource.java | 54 +-
.../function/valuesource/FieldCacheSource.java | 11 +-
.../function/valuesource/FloatFieldSource.java | 29 +-
.../function/valuesource/IDFValueSource.java | 35 +-
.../queries/function/valuesource/IfFunction.java | 37 +-
.../function/valuesource/IntFieldSource.java | 24 +-
.../valuesource/JoinDocFreqValueSource.java | 24 +-
.../function/valuesource/LinearFloatFunction.java | 27 +-
.../function/valuesource/LiteralValueSource.java | 18 +-
.../function/valuesource/LongFieldSource.java | 25 +-
.../function/valuesource/MaxDocValueSource.java | 17 +-
.../function/valuesource/MaxFloatFunction.java | 15 +-
.../function/valuesource/MinFloatFunction.java | 18 +-
.../function/valuesource/MultiBoolFunction.java | 16 +-
.../function/valuesource/MultiFloatFunction.java | 45 +-
.../function/valuesource/MultiFunction.java | 76 +-
.../function/valuesource/MultiValueSource.java | 8 +-
.../valuesource/MultiValuedDoubleFieldSource.java | 24 +-
.../valuesource/MultiValuedFloatFieldSource.java | 24 +-
.../valuesource/MultiValuedIntFieldSource.java | 24 +-
.../valuesource/MultiValuedLongFieldSource.java | 24 +-
.../function/valuesource/NormValueSource.java | 47 +-
.../function/valuesource/NumDocsValueSource.java | 18 +-
.../function/valuesource/PowFloatFunction.java | 16 +-
.../function/valuesource/ProductFloatFunction.java | 5 +-
.../function/valuesource/QueryValueSource.java | 52 +-
.../valuesource/RangeMapFloatFunction.java | 75 +-
.../valuesource/ReciprocalFloatFunction.java | 61 +-
.../function/valuesource/ScaleFloatFunction.java | 75 +-
.../function/valuesource/SimpleBoolFunction.java | 21 +-
.../function/valuesource/SimpleFloatFunction.java | 16 +-
.../function/valuesource/SingleFunction.java | 17 +-
.../function/valuesource/SortedSetFieldSource.java | 22 +-
.../function/valuesource/SumFloatFunction.java | 7 +-
.../valuesource/SumTotalTermFreqValueSource.java | 31 +-
.../function/valuesource/TFValueSource.java | 131 +-
.../function/valuesource/TermFreqValueSource.java | 119 +-
.../valuesource/TotalTermFreqValueSource.java | 39 +-
.../function/valuesource/VectorValueSource.java | 35 +-
.../queries/function/valuesource/package-info.java | 4 +-
.../queries/intervals/BlockIntervalsSource.java | 14 +-
.../queries/intervals/CachingMatchesIterator.java | 17 +-
.../lucene/queries/intervals/ConjunctionDISI.java | 16 +-
.../intervals/ConjunctionIntervalIterator.java | 2 -
.../intervals/ConjunctionIntervalsSource.java | 21 +-
.../intervals/ContainedByIntervalsSource.java | 6 +-
.../intervals/ContainingIntervalsSource.java | 6 +-
.../intervals/DifferenceIntervalsSource.java | 16 +-
.../queries/intervals/DisiPriorityQueue.java | 15 +-
.../lucene/queries/intervals/DisiWrapper.java | 1 -
.../intervals/DisjunctionDISIApproximation.java | 8 +-
.../intervals/DisjunctionIntervalsSource.java | 195 +-
.../lucene/queries/intervals/Disjunctions.java | 15 +-
.../intervals/ExtendedIntervalIterator.java | 15 +-
.../queries/intervals/ExtendedIntervalsSource.java | 15 +-
.../queries/intervals/FilteredIntervalsSource.java | 38 +-
.../intervals/FixedFieldIntervalsSource.java | 11 +-
.../lucene/queries/intervals/IntervalFilter.java | 15 +-
.../lucene/queries/intervals/IntervalIterator.java | 54 +-
.../lucene/queries/intervals/IntervalMatches.java | 11 +-
.../queries/intervals/IntervalMatchesIterator.java | 10 +-
.../lucene/queries/intervals/IntervalQuery.java | 102 +-
.../queries/intervals/IntervalScoreFunction.java | 27 +-
.../lucene/queries/intervals/IntervalScorer.java | 12 +-
.../apache/lucene/queries/intervals/Intervals.java | 222 +-
.../lucene/queries/intervals/IntervalsSource.java | 43 +-
.../MinimizingConjunctionMatchesIterator.java | 6 +-
.../MinimumShouldMatchIntervalsSource.java | 54 +-
.../intervals/MultiTermIntervalsSource.java | 28 +-
.../intervals/NonOverlappingIntervalsSource.java | 6 +-
.../intervals/NotContainedByIntervalsSource.java | 12 +-
.../intervals/NotContainingIntervalsSource.java | 10 +-
.../queries/intervals/OffsetIntervalsSource.java | 19 +-
.../queries/intervals/OrderedIntervalsSource.java | 32 +-
.../intervals/OverlappingIntervalsSource.java | 10 +-
.../PayloadFilteredTermIntervalsSource.java | 43 +-
.../lucene/queries/intervals/RelativeIterator.java | 3 +-
.../intervals/RepeatingIntervalsSource.java | 24 +-
.../queries/intervals/TermIntervalsSource.java | 64 +-
.../intervals/UnorderedIntervalsSource.java | 30 +-
.../lucene/queries/intervals/package-info.java | 65 +-
.../apache/lucene/queries/mlt/MoreLikeThis.java | 366 +--
.../lucene/queries/mlt/MoreLikeThisQuery.java | 42 +-
.../apache/lucene/queries/mlt/package-info.java | 4 +-
.../org/apache/lucene/queries/package-info.java | 4 +-
.../queries/payloads/AveragePayloadFunction.java | 25 +-
.../queries/payloads/MaxPayloadFunction.java | 25 +-
.../queries/payloads/MinPayloadFunction.java | 26 +-
.../lucene/queries/payloads/PayloadDecoder.java | 13 +-
.../lucene/queries/payloads/PayloadFunction.java | 37 +-
.../lucene/queries/payloads/PayloadScoreQuery.java | 86 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 67 +-
.../queries/payloads/SumPayloadFunction.java | 23 +-
.../lucene/queries/payloads/package-info.java | 16 +-
.../lucene/queries/TestCommonTermsQuery.java | 308 +-
.../lucene/queries/function/FunctionTestSetup.java | 101 +-
.../function/TestDocValuesFieldSources.java | 18 +-
.../queries/function/TestFieldScoreQuery.java | 72 +-
.../queries/function/TestFunctionMatchQuery.java | 12 +-
.../function/TestFunctionQueryExplanations.java | 16 +-
.../queries/function/TestFunctionQuerySort.java | 18 +-
.../queries/function/TestFunctionRangeQuery.java | 44 +-
.../function/TestFunctionScoreExplanations.java | 11 +-
.../queries/function/TestFunctionScoreQuery.java | 170 +-
.../queries/function/TestIndexReaderFunctions.java | 80 +-
.../queries/function/TestLongNormValueSource.java | 6 +-
.../queries/function/TestSortedSetFieldSource.java | 14 +-
.../lucene/queries/function/TestValueSources.java | 562 ++--
.../docvalues/TestBoolValOfNumericDVs.java | 72 +-
.../queries/intervals/OneTimeIntervalSource.java | 14 +-
.../queries/intervals/TestDisjunctionRewrites.java | 213 +-
.../queries/intervals/TestIntervalQuery.java | 361 +-
.../lucene/queries/intervals/TestIntervals.java | 869 +++--
.../intervals/TestPayloadFilteredInterval.java | 33 +-
.../queries/intervals/TestSimplifications.java | 44 +-
.../lucene/queries/mlt/TestMoreLikeThis.java | 280 +-
.../lucene/queries/payloads/PayloadHelper.java | 54 +-
.../queries/payloads/TestPayloadCheckQuery.java | 104 +-
.../queries/payloads/TestPayloadExplanations.java | 69 +-
.../queries/payloads/TestPayloadScoreQuery.java | 217 +-
.../lucene/queries/payloads/TestPayloadSpans.java | 226 +-
.../queries/payloads/TestPayloadTermQuery.java | 152 +-
.../org/apache/lucene/spatial/SpatialTestCase.java | 4 -
.../solr/handler/FieldAnalysisRequestHandler.java | 2 -
898 files changed, 44596 insertions(+), 39640 deletions(-)
[lucene-solr] 03/03: LUCENE-9570: code reformatting [record rev].
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 4160ceda2eea54d15ff7764f66546aaac4859e76
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Mon Dec 28 12:26:41 2020 +0100
LUCENE-9570: code reformatting [record rev].
---
.git-blame-ignore-revs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index a843af5..ad75695 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -1,3 +1,4 @@
# LUCENE-9570 [google java formatter applied]
6faa4f98e04817bf27d02d96332904a29fb7db69
-2d6ad2fee6dfd96388594f4de9b37c037efe8017
\ No newline at end of file
+2d6ad2fee6dfd96388594f4de9b37c037efe8017
+8ef6a0da56878177ff8d6880c92e8f7d0321d076
\ No newline at end of file
[lucene-solr] 02/03: LUCENE-9570: code reformatting [partial].
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 8ef6a0da56878177ff8d6880c92e8f7d0321d076
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Mon Dec 28 12:26:13 2020 +0100
LUCENE-9570: code reformatting [partial].
---
gradle/generation/jflex.gradle | 46 +-
gradle/validation/spotless.gradle | 43 +-
.../lucene/analysis/bg/BulgarianAnalyzer.java | 76 +-
.../lucene/analysis/bg/BulgarianStemFilter.java | 24 +-
.../analysis/bg/BulgarianStemFilterFactory.java | 12 +-
.../lucene/analysis/bg/BulgarianStemmer.java | 86 +-
.../apache/lucene/analysis/bg/package-info.java | 4 +-
.../apache/lucene/analysis/bn/BengaliAnalyzer.java | 65 +-
.../analysis/bn/BengaliNormalizationFilter.java | 25 +-
.../bn/BengaliNormalizationFilterFactory.java | 14 +-
.../lucene/analysis/bn/BengaliNormalizer.java | 72 +-
.../lucene/analysis/bn/BengaliStemFilter.java | 15 +-
.../analysis/bn/BengaliStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/bn/BengaliStemmer.java | 255 +-
.../apache/lucene/analysis/bn/package-info.java | 4 +-
.../analysis/boost/DelimitedBoostTokenFilter.java | 14 +-
.../boost/DelimitedBoostTokenFilterFactory.java | 16 +-
.../apache/lucene/analysis/boost/package-info.java | 4 +-
.../lucene/analysis/br/BrazilianAnalyzer.java | 72 +-
.../lucene/analysis/br/BrazilianStemFilter.java | 31 +-
.../analysis/br/BrazilianStemFilterFactory.java | 14 +-
.../lucene/analysis/br/BrazilianStemmer.java | 1202 ++++---
.../apache/lucene/analysis/br/package-info.java | 4 +-
.../apache/lucene/analysis/ca/CatalanAnalyzer.java | 66 +-
.../apache/lucene/analysis/ca/package-info.java | 4 +-
.../lucene/analysis/charfilter/BaseCharFilter.java | 49 +-
.../charfilter/HTMLStripCharFilterFactory.java | 11 +-
.../analysis/charfilter/MappingCharFilter.java | 26 +-
.../charfilter/MappingCharFilterFactory.java | 80 +-
.../analysis/charfilter/NormalizeCharMap.java | 46 +-
.../lucene/analysis/charfilter/package-info.java | 61 +-
.../apache/lucene/analysis/cjk/CJKAnalyzer.java | 35 +-
.../lucene/analysis/cjk/CJKBigramFilter.java | 191 +-
.../analysis/cjk/CJKBigramFilterFactory.java | 16 +-
.../lucene/analysis/cjk/CJKWidthCharFilter.java | 67 +-
.../analysis/cjk/CJKWidthCharFilterFactory.java | 8 +-
.../apache/lucene/analysis/cjk/CJKWidthFilter.java | 75 +-
.../lucene/analysis/cjk/CJKWidthFilterFactory.java | 14 +-
.../apache/lucene/analysis/cjk/package-info.java | 29 +-
.../apache/lucene/analysis/ckb/SoraniAnalyzer.java | 57 +-
.../analysis/ckb/SoraniNormalizationFilter.java | 9 +-
.../ckb/SoraniNormalizationFilterFactory.java | 8 +-
.../lucene/analysis/ckb/SoraniNormalizer.java | 28 +-
.../lucene/analysis/ckb/SoraniStemFilter.java | 24 +-
.../analysis/ckb/SoraniStemFilterFactory.java | 8 +-
.../apache/lucene/analysis/ckb/SoraniStemmer.java | 52 +-
.../apache/lucene/analysis/ckb/package-info.java | 4 +-
.../lucene/analysis/classic/ClassicAnalyzer.java | 60 +-
.../lucene/analysis/classic/ClassicFilter.java | 28 +-
.../analysis/classic/ClassicFilterFactory.java | 12 +-
.../lucene/analysis/classic/ClassicTokenizer.java | 99 +-
.../analysis/classic/ClassicTokenizerFactory.java | 7 +-
.../analysis/classic/ClassicTokenizerImpl.java | 706 ++--
.../analysis/classic/ClassicTokenizerImpl.jflex | 2 +-
.../lucene/analysis/classic/package-info.java | 19 +-
.../analysis/commongrams/CommonGramsFilter.java | 80 +-
.../commongrams/CommonGramsFilterFactory.java | 12 +-
.../commongrams/CommonGramsQueryFilter.java | 62 +-
.../commongrams/CommonGramsQueryFilterFactory.java | 10 +-
.../lucene/analysis/commongrams/package-info.java | 4 +-
.../compound/CompoundWordTokenFilterBase.java | 78 +-
.../DictionaryCompoundWordTokenFilter.java | 89 +-
.../DictionaryCompoundWordTokenFilterFactory.java | 24 +-
.../HyphenationCompoundWordTokenFilter.java | 118 +-
.../HyphenationCompoundWordTokenFilterFactory.java | 58 +-
.../analysis/compound/hyphenation/ByteVector.java | 31 +-
.../analysis/compound/hyphenation/CharVector.java | 33 +-
.../analysis/compound/hyphenation/Hyphen.java | 24 +-
.../analysis/compound/hyphenation/Hyphenation.java | 17 +-
.../compound/hyphenation/HyphenationTree.java | 179 +-
.../compound/hyphenation/PatternConsumer.java | 25 +-
.../compound/hyphenation/PatternParser.java | 67 +-
.../analysis/compound/hyphenation/TernaryTree.java | 183 +-
.../compound/hyphenation/package-info.java | 9 +-
.../lucene/analysis/compound/package-info.java | 123 +-
.../lucene/analysis/core/DecimalDigitFilter.java | 16 +-
.../analysis/core/DecimalDigitFilterFactory.java | 12 +-
.../lucene/analysis/core/FlattenGraphFilter.java | 143 +-
.../analysis/core/FlattenGraphFilterFactory.java | 11 +-
.../lucene/analysis/core/KeywordAnalyzer.java | 8 +-
.../lucene/analysis/core/KeywordTokenizer.java | 37 +-
.../analysis/core/KeywordTokenizerFactory.java | 31 +-
.../lucene/analysis/core/LetterTokenizer.java | 47 +-
.../analysis/core/LetterTokenizerFactory.java | 26 +-
.../lucene/analysis/core/LowerCaseFilter.java | 13 +-
.../analysis/core/LowerCaseFilterFactory.java | 11 +-
.../lucene/analysis/core/SimpleAnalyzer.java | 16 +-
.../apache/lucene/analysis/core/StopAnalyzer.java | 41 +-
.../apache/lucene/analysis/core/StopFilter.java | 20 +-
.../lucene/analysis/core/StopFilterFactory.java | 62 +-
.../lucene/analysis/core/TypeTokenFilter.java | 24 +-
.../analysis/core/TypeTokenFilterFactory.java | 18 +-
.../analysis/core/UnicodeWhitespaceAnalyzer.java | 14 +-
.../analysis/core/UnicodeWhitespaceTokenizer.java | 37 +-
.../lucene/analysis/core/UpperCaseFilter.java | 20 +-
.../analysis/core/UpperCaseFilterFactory.java | 21 +-
.../lucene/analysis/core/WhitespaceAnalyzer.java | 12 +-
.../lucene/analysis/core/WhitespaceTokenizer.java | 38 +-
.../analysis/core/WhitespaceTokenizerFactory.java | 28 +-
.../apache/lucene/analysis/core/package-info.java | 4 +-
.../lucene/analysis/custom/CustomAnalyzer.java | 467 +--
.../lucene/analysis/custom/package-info.java | 4 +-
.../apache/lucene/analysis/cz/CzechAnalyzer.java | 61 +-
.../apache/lucene/analysis/cz/CzechStemFilter.java | 24 +-
.../lucene/analysis/cz/CzechStemFilterFactory.java | 12 +-
.../apache/lucene/analysis/cz/CzechStemmer.java | 141 +-
.../apache/lucene/analysis/cz/package-info.java | 4 +-
.../apache/lucene/analysis/da/DanishAnalyzer.java | 57 +-
.../apache/lucene/analysis/da/package-info.java | 4 +-
.../apache/lucene/analysis/de/GermanAnalyzer.java | 93 +-
.../lucene/analysis/de/GermanLightStemFilter.java | 16 +-
.../analysis/de/GermanLightStemFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanLightStemmer.java | 125 +-
.../analysis/de/GermanMinimalStemFilter.java | 16 +-
.../de/GermanMinimalStemFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanMinimalStemmer.java | 99 +-
.../analysis/de/GermanNormalizationFilter.java | 37 +-
.../de/GermanNormalizationFilterFactory.java | 12 +-
.../lucene/analysis/de/GermanStemFilter.java | 93 +-
.../analysis/de/GermanStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/de/GermanStemmer.java | 412 ++-
.../apache/lucene/analysis/de/package-info.java | 4 +-
.../apache/lucene/analysis/el/GreekAnalyzer.java | 66 +-
.../lucene/analysis/el/GreekLowerCaseFilter.java | 66 +-
.../analysis/el/GreekLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/el/GreekStemFilter.java | 33 +-
.../lucene/analysis/el/GreekStemFilterFactory.java | 14 +-
.../apache/lucene/analysis/el/GreekStemmer.java | 1091 +++---
.../apache/lucene/analysis/el/package-info.java | 4 +-
.../analysis/email/UAX29URLEmailAnalyzer.java | 59 +-
.../analysis/email/UAX29URLEmailTokenizer.java | 80 +-
.../email/UAX29URLEmailTokenizerFactory.java | 11 +-
.../apache/lucene/analysis/email/package-info.java | 30 +-
.../apache/lucene/analysis/en/EnglishAnalyzer.java | 61 +-
.../analysis/en/EnglishMinimalStemFilter.java | 16 +-
.../en/EnglishMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/en/EnglishMinimalStemmer.java | 24 +-
.../analysis/en/EnglishPossessiveFilter.java | 20 +-
.../en/EnglishPossessiveFilterFactory.java | 14 +-
.../org/apache/lucene/analysis/en/KStemData1.java | 1412 ++++----
.../org/apache/lucene/analysis/en/KStemData2.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData3.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData4.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData5.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData6.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData7.java | 1411 ++++----
.../org/apache/lucene/analysis/en/KStemData8.java | 1209 ++++---
.../org/apache/lucene/analysis/en/KStemFilter.java | 41 +-
.../lucene/analysis/en/KStemFilterFactory.java | 10 +-
.../org/apache/lucene/analysis/en/KStemmer.java | 1265 ++++---
.../lucene/analysis/en/PorterStemFilter.java | 62 +-
.../analysis/en/PorterStemFilterFactory.java | 11 +-
.../apache/lucene/analysis/en/PorterStemmer.java | 500 +--
.../apache/lucene/analysis/en/package-info.java | 4 +-
.../apache/lucene/analysis/es/SpanishAnalyzer.java | 58 +-
.../lucene/analysis/es/SpanishLightStemFilter.java | 16 +-
.../analysis/es/SpanishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/es/SpanishLightStemmer.java | 92 +-
.../analysis/es/SpanishMinimalStemFilter.java | 13 +-
.../es/SpanishMinimalStemFilterFactory.java | 11 +-
.../lucene/analysis/es/SpanishMinimalStemmer.java | 55 +-
.../apache/lucene/analysis/es/package-info.java | 4 +-
.../lucene/analysis/et/EstonianAnalyzer.java | 155 +-
.../apache/lucene/analysis/et/package-info.java | 4 +-
.../apache/lucene/analysis/eu/BasqueAnalyzer.java | 54 +-
.../apache/lucene/analysis/eu/package-info.java | 4 +-
.../apache/lucene/analysis/fa/PersianAnalyzer.java | 77 +-
.../lucene/analysis/fa/PersianCharFilter.java | 14 +-
.../analysis/fa/PersianCharFilterFactory.java | 7 +-
.../analysis/fa/PersianNormalizationFilter.java | 14 +-
.../fa/PersianNormalizationFilterFactory.java | 14 +-
.../lucene/analysis/fa/PersianNormalizer.java | 53 +-
.../apache/lucene/analysis/fa/package-info.java | 4 +-
.../apache/lucene/analysis/fi/FinnishAnalyzer.java | 57 +-
.../lucene/analysis/fi/FinnishLightStemFilter.java | 16 +-
.../analysis/fi/FinnishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/fi/FinnishLightStemmer.java | 237 +-
.../apache/lucene/analysis/fi/package-info.java | 4 +-
.../apache/lucene/analysis/fr/FrenchAnalyzer.java | 99 +-
.../lucene/analysis/fr/FrenchLightStemFilter.java | 16 +-
.../analysis/fr/FrenchLightStemFilterFactory.java | 12 +-
.../lucene/analysis/fr/FrenchLightStemmer.java | 261 +-
.../analysis/fr/FrenchMinimalStemFilter.java | 16 +-
.../fr/FrenchMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/fr/FrenchMinimalStemmer.java | 66 +-
.../apache/lucene/analysis/fr/package-info.java | 4 +-
.../apache/lucene/analysis/ga/IrishAnalyzer.java | 79 +-
.../lucene/analysis/ga/IrishLowerCaseFilter.java | 22 +-
.../analysis/ga/IrishLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/ga/package-info.java | 4 +-
.../lucene/analysis/gl/GalicianAnalyzer.java | 57 +-
.../analysis/gl/GalicianMinimalStemFilter.java | 16 +-
.../gl/GalicianMinimalStemFilterFactory.java | 14 +-
.../lucene/analysis/gl/GalicianMinimalStemmer.java | 17 +-
.../lucene/analysis/gl/GalicianStemFilter.java | 18 +-
.../analysis/gl/GalicianStemFilterFactory.java | 14 +-
.../apache/lucene/analysis/gl/GalicianStemmer.java | 47 +-
.../apache/lucene/analysis/gl/package-info.java | 4 +-
.../apache/lucene/analysis/hi/HindiAnalyzer.java | 65 +-
.../analysis/hi/HindiNormalizationFilter.java | 27 +-
.../hi/HindiNormalizationFilterFactory.java | 16 +-
.../apache/lucene/analysis/hi/HindiNormalizer.java | 271 +-
.../apache/lucene/analysis/hi/HindiStemFilter.java | 12 +-
.../lucene/analysis/hi/HindiStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/hi/HindiStemmer.java | 166 +-
.../apache/lucene/analysis/hi/package-info.java | 4 +-
.../lucene/analysis/hu/HungarianAnalyzer.java | 57 +-
.../analysis/hu/HungarianLightStemFilter.java | 16 +-
.../hu/HungarianLightStemFilterFactory.java | 16 +-
.../lucene/analysis/hu/HungarianLightStemmer.java | 252 +-
.../apache/lucene/analysis/hu/package-info.java | 4 +-
.../lucene/analysis/hunspell/Dictionary.java | 518 +--
.../analysis/hunspell/HunspellStemFilter.java | 96 +-
.../hunspell/HunspellStemFilterFactory.java | 45 +-
.../analysis/hunspell/ISO8859_14Decoder.java | 32 +-
.../apache/lucene/analysis/hunspell/Stemmer.java | 329 +-
.../lucene/analysis/hunspell/package-info.java | 11 +-
.../lucene/analysis/hy/ArmenianAnalyzer.java | 55 +-
.../apache/lucene/analysis/hy/package-info.java | 4 +-
.../lucene/analysis/id/IndonesianAnalyzer.java | 62 +-
.../lucene/analysis/id/IndonesianStemFilter.java | 23 +-
.../analysis/id/IndonesianStemFilterFactory.java | 12 +-
.../lucene/analysis/id/IndonesianStemmer.java | 130 +-
.../apache/lucene/analysis/id/package-info.java | 4 +-
.../analysis/in/IndicNormalizationFilter.java | 9 +-
.../in/IndicNormalizationFilterFactory.java | 16 +-
.../apache/lucene/analysis/in/IndicNormalizer.java | 407 ++-
.../apache/lucene/analysis/in/package-info.java | 4 +-
.../apache/lucene/analysis/it/ItalianAnalyzer.java | 71 +-
.../lucene/analysis/it/ItalianLightStemFilter.java | 16 +-
.../analysis/it/ItalianLightStemFilterFactory.java | 18 +-
.../lucene/analysis/it/ItalianLightStemmer.java | 103 +-
.../apache/lucene/analysis/it/package-info.java | 4 +-
.../lucene/analysis/lt/LithuanianAnalyzer.java | 55 +-
.../apache/lucene/analysis/lt/package-info.java | 4 +-
.../apache/lucene/analysis/lv/LatvianAnalyzer.java | 57 +-
.../lucene/analysis/lv/LatvianStemFilter.java | 16 +-
.../analysis/lv/LatvianStemFilterFactory.java | 16 +-
.../apache/lucene/analysis/lv/LatvianStemmer.java | 155 +-
.../apache/lucene/analysis/lv/package-info.java | 4 +-
.../lucene/analysis/minhash/MinHashFilter.java | 73 +-
.../analysis/minhash/MinHashFilterFactory.java | 17 +-
.../lucene/analysis/minhash/package-info.java | 4 +-
.../analysis/miscellaneous/ASCIIFoldingFilter.java | 134 +-
.../miscellaneous/ASCIIFoldingFilterFactory.java | 15 +-
.../miscellaneous/CapitalizationFilter.java | 73 +-
.../miscellaneous/CapitalizationFilterFactory.java | 52 +-
.../miscellaneous/CodepointCountFilter.java | 21 +-
.../miscellaneous/CodepointCountFilterFactory.java | 10 +-
.../miscellaneous/ConcatenateGraphFilter.java | 150 +-
.../ConcatenateGraphFilterFactory.java | 59 +-
.../miscellaneous/ConcatenatingTokenStream.java | 30 +-
.../miscellaneous/ConditionalTokenFilter.java | 40 +-
.../ConditionalTokenFilterFactory.java | 41 +-
.../miscellaneous/DateRecognizerFilter.java | 11 +-
.../miscellaneous/DateRecognizerFilterFactory.java | 21 +-
.../DelimitedTermFrequencyTokenFilter.java | 23 +-
.../DelimitedTermFrequencyTokenFilterFactory.java | 8 +-
.../miscellaneous/DropIfFlaggedFilter.java | 1 -
.../miscellaneous/DropIfFlaggedFilterFactory.java | 21 +-
.../analysis/miscellaneous/EmptyTokenStream.java | 6 +-
.../analysis/miscellaneous/FingerprintFilter.java | 93 +-
.../miscellaneous/FingerprintFilterFactory.java | 19 +-
.../miscellaneous/FixBrokenOffsetsFilter.java | 3 +-
.../FixBrokenOffsetsFilterFactory.java | 6 +-
.../miscellaneous/HyphenatedWordsFilter.java | 34 +-
.../HyphenatedWordsFilterFactory.java | 13 +-
.../analysis/miscellaneous/KeepWordFilter.java | 16 +-
.../miscellaneous/KeepWordFilterFactory.java | 11 +-
.../miscellaneous/KeywordMarkerFilter.java | 10 +-
.../miscellaneous/KeywordMarkerFilterFactory.java | 21 +-
.../miscellaneous/KeywordRepeatFilter.java | 20 +-
.../miscellaneous/KeywordRepeatFilterFactory.java | 17 +-
.../analysis/miscellaneous/LengthFilter.java | 21 +-
.../miscellaneous/LengthFilterFactory.java | 11 +-
.../miscellaneous/LimitTokenCountAnalyzer.java | 33 +-
.../miscellaneous/LimitTokenCountFilter.java | 38 +-
.../LimitTokenCountFilterFactory.java | 15 +-
.../miscellaneous/LimitTokenOffsetFilter.java | 23 +-
.../LimitTokenOffsetFilterFactory.java | 10 +-
.../miscellaneous/LimitTokenPositionFilter.java | 43 +-
.../LimitTokenPositionFilterFactory.java | 16 +-
.../miscellaneous/PatternKeywordMarkerFilter.java | 27 +-
.../miscellaneous/PerFieldAnalyzerWrapper.java | 57 +-
.../miscellaneous/ProtectedTermFilter.java | 18 +-
.../miscellaneous/ProtectedTermFilterFactory.java | 72 +-
.../miscellaneous/RemoveDuplicatesTokenFilter.java | 23 +-
.../RemoveDuplicatesTokenFilterFactory.java | 11 +-
.../miscellaneous/ScandinavianFoldingFilter.java | 81 +-
.../ScandinavianFoldingFilterFactory.java | 8 +-
.../ScandinavianNormalizationFilter.java | 55 +-
.../ScandinavianNormalizationFilterFactory.java | 6 +-
.../miscellaneous/SetKeywordMarkerFilter.java | 23 +-
.../miscellaneous/StemmerOverrideFilter.java | 95 +-
.../StemmerOverrideFilterFactory.java | 11 +-
.../lucene/analysis/miscellaneous/TrimFilter.java | 23 +-
.../analysis/miscellaneous/TrimFilterFactory.java | 12 +-
.../miscellaneous/TruncateTokenFilter.java | 16 +-
.../miscellaneous/TruncateTokenFilterFactory.java | 14 +-
.../miscellaneous/TypeAsSynonymFilter.java | 28 +-
.../miscellaneous/TypeAsSynonymFilterFactory.java | 24 +-
.../miscellaneous/WordDelimiterFilter.java | 267 +-
.../miscellaneous/WordDelimiterFilterFactory.java | 134 +-
.../miscellaneous/WordDelimiterGraphFilter.java | 325 +-
.../WordDelimiterGraphFilterFactory.java | 135 +-
.../miscellaneous/WordDelimiterIterator.java | 184 +-
.../analysis/miscellaneous/package-info.java | 6 +-
.../analysis/ngram/EdgeNGramFilterFactory.java | 8 +-
.../analysis/ngram/EdgeNGramTokenFilter.java | 42 +-
.../lucene/analysis/ngram/EdgeNGramTokenizer.java | 12 +-
.../analysis/ngram/EdgeNGramTokenizerFactory.java | 7 +-
.../lucene/analysis/ngram/NGramFilterFactory.java | 8 +-
.../lucene/analysis/ngram/NGramTokenFilter.java | 71 +-
.../lucene/analysis/ngram/NGramTokenizer.java | 64 +-
.../analysis/ngram/NGramTokenizerFactory.java | 14 +-
.../apache/lucene/analysis/ngram/package-info.java | 4 +-
.../apache/lucene/analysis/nl/DutchAnalyzer.java | 92 +-
.../apache/lucene/analysis/nl/package-info.java | 4 +-
.../lucene/analysis/no/NorwegianAnalyzer.java | 58 +-
.../analysis/no/NorwegianLightStemFilter.java | 33 +-
.../no/NorwegianLightStemFilterFactory.java | 20 +-
.../lucene/analysis/no/NorwegianLightStemmer.java | 224 +-
.../analysis/no/NorwegianMinimalStemFilter.java | 31 +-
.../no/NorwegianMinimalStemFilterFactory.java | 20 +-
.../analysis/no/NorwegianMinimalStemmer.java | 95 +-
.../apache/lucene/analysis/no/package-info.java | 4 +-
.../analysis/path/PathHierarchyTokenizer.java | 76 +-
.../path/PathHierarchyTokenizerFactory.java | 48 +-
.../path/ReversePathHierarchyTokenizer.java | 45 +-
.../apache/lucene/analysis/path/package-info.java | 4 +-
.../pattern/PatternCaptureGroupFilterFactory.java | 18 +-
.../pattern/PatternCaptureGroupTokenFilter.java | 86 +-
.../analysis/pattern/PatternReplaceCharFilter.java | 49 +-
.../pattern/PatternReplaceCharFilterFactory.java | 9 +-
.../analysis/pattern/PatternReplaceFilter.java | 36 +-
.../pattern/PatternReplaceFilterFactory.java | 12 +-
.../lucene/analysis/pattern/PatternTokenizer.java | 61 +-
.../analysis/pattern/PatternTokenizerFactory.java | 52 +-
.../pattern/SimplePatternSplitTokenizer.java | 43 +-
.../SimplePatternSplitTokenizerFactory.java | 45 +-
.../analysis/pattern/SimplePatternTokenizer.java | 55 +-
.../pattern/SimplePatternTokenizerFactory.java | 40 +-
.../lucene/analysis/pattern/package-info.java | 4 +-
.../lucene/analysis/payloads/AbstractEncoder.java | 8 +-
.../payloads/DelimitedPayloadTokenFilter.java | 24 +-
.../DelimitedPayloadTokenFilterFactory.java | 18 +-
.../lucene/analysis/payloads/FloatEncoder.java | 8 +-
.../lucene/analysis/payloads/IdentityEncoder.java | 14 +-
.../lucene/analysis/payloads/IntegerEncoder.java | 13 +-
.../payloads/NumericPayloadTokenFilter.java | 13 +-
.../payloads/NumericPayloadTokenFilterFactory.java | 13 +-
.../lucene/analysis/payloads/PayloadEncoder.java | 13 +-
.../lucene/analysis/payloads/PayloadHelper.java | 36 +-
.../payloads/TokenOffsetPayloadTokenFilter.java | 15 +-
.../TokenOffsetPayloadTokenFilterFactory.java | 14 +-
.../payloads/TypeAsPayloadTokenFilter.java | 11 +-
.../payloads/TypeAsPayloadTokenFilterFactory.java | 14 +-
.../lucene/analysis/payloads/package-info.java | 4 +-
.../lucene/analysis/pt/PortugueseAnalyzer.java | 58 +-
.../analysis/pt/PortugueseLightStemFilter.java | 16 +-
.../pt/PortugueseLightStemFilterFactory.java | 14 +-
.../lucene/analysis/pt/PortugueseLightStemmer.java | 162 +-
.../analysis/pt/PortugueseMinimalStemFilter.java | 16 +-
.../pt/PortugueseMinimalStemFilterFactory.java | 14 +-
.../analysis/pt/PortugueseMinimalStemmer.java | 22 +-
.../lucene/analysis/pt/PortugueseStemFilter.java | 18 +-
.../analysis/pt/PortugueseStemFilterFactory.java | 16 +-
.../lucene/analysis/pt/PortugueseStemmer.java | 65 +-
.../apache/lucene/analysis/pt/RSLPStemmerBase.java | 262 +-
.../apache/lucene/analysis/pt/package-info.java | 4 +-
.../analysis/query/QueryAutoStopWordAnalyzer.java | 90 +-
.../apache/lucene/analysis/query/package-info.java | 4 +-
.../analysis/reverse/ReverseStringFilter.java | 106 +-
.../reverse/ReverseStringFilterFactory.java | 12 +-
.../lucene/analysis/reverse/package-info.java | 4 +-
.../lucene/analysis/ro/RomanianAnalyzer.java | 61 +-
.../apache/lucene/analysis/ro/package-info.java | 4 +-
.../apache/lucene/analysis/ru/RussianAnalyzer.java | 159 +-
.../lucene/analysis/ru/RussianLightStemFilter.java | 16 +-
.../analysis/ru/RussianLightStemFilterFactory.java | 14 +-
.../lucene/analysis/ru/RussianLightStemmer.java | 168 +-
.../apache/lucene/analysis/ru/package-info.java | 4 +-
.../analysis/shingle/FixedShingleFilter.java | 45 +-
.../shingle/FixedShingleFilterFactory.java | 6 +-
.../analysis/shingle/ShingleAnalyzerWrapper.java | 52 +-
.../lucene/analysis/shingle/ShingleFilter.java | 295 +-
.../analysis/shingle/ShingleFilterFactory.java | 24 +-
.../lucene/analysis/shingle/package-info.java | 4 +-
.../lucene/analysis/sinks/TeeSinkTokenFilter.java | 44 +-
.../apache/lucene/analysis/sinks/package-info.java | 4 +-
.../lucene/analysis/snowball/SnowballFilter.java | 42 +-
.../snowball/SnowballPorterFilterFactory.java | 20 +-
.../lucene/analysis/snowball/package-info.java | 53 +-
.../apache/lucene/analysis/sr/SerbianAnalyzer.java | 49 +-
.../analysis/sr/SerbianNormalizationFilter.java | 243 +-
.../sr/SerbianNormalizationFilterFactory.java | 16 +-
.../sr/SerbianNormalizationRegularFilter.java | 228 +-
.../apache/lucene/analysis/sr/package-info.java | 4 +-
.../apache/lucene/analysis/sv/SwedishAnalyzer.java | 57 +-
.../lucene/analysis/sv/SwedishLightStemFilter.java | 16 +-
.../analysis/sv/SwedishLightStemFilterFactory.java | 14 +-
.../lucene/analysis/sv/SwedishLightStemmer.java | 112 +-
.../apache/lucene/analysis/sv/package-info.java | 4 +-
.../lucene/analysis/synonym/SolrSynonymParser.java | 79 +-
.../lucene/analysis/synonym/SynonymFilter.java | 193 +-
.../analysis/synonym/SynonymFilterFactory.java | 114 +-
.../analysis/synonym/SynonymGraphFilter.java | 185 +-
.../synonym/SynonymGraphFilterFactory.java | 110 +-
.../apache/lucene/analysis/synonym/SynonymMap.java | 128 +-
.../analysis/synonym/WordnetSynonymParser.java | 28 +-
.../lucene/analysis/synonym/package-info.java | 4 +-
.../apache/lucene/analysis/th/ThaiAnalyzer.java | 44 +-
.../apache/lucene/analysis/th/ThaiTokenizer.java | 45 +-
.../lucene/analysis/th/ThaiTokenizerFactory.java | 12 +-
.../apache/lucene/analysis/th/package-info.java | 4 +-
.../lucene/analysis/tr/ApostropheFilter.java | 20 +-
.../analysis/tr/ApostropheFilterFactory.java | 8 +-
.../apache/lucene/analysis/tr/TurkishAnalyzer.java | 54 +-
.../lucene/analysis/tr/TurkishLowerCaseFilter.java | 74 +-
.../analysis/tr/TurkishLowerCaseFilterFactory.java | 14 +-
.../apache/lucene/analysis/tr/package-info.java | 4 +-
.../lucene/analysis/util/CharArrayIterator.java | 45 +-
.../apache/lucene/analysis/util/CharTokenizer.java | 127 +-
.../apache/lucene/analysis/util/ElisionFilter.java | 15 +-
.../lucene/analysis/util/ElisionFilterFactory.java | 12 +-
.../analysis/util/FilesystemResourceLoader.java | 55 +-
.../lucene/analysis/util/OpenStringBuilder.java | 55 +-
.../lucene/analysis/util/RollingCharBuffer.java | 46 +-
.../analysis/util/SegmentingTokenizerBase.java | 97 +-
.../apache/lucene/analysis/util/StemmerUtil.java | 44 +-
.../apache/lucene/analysis/util/UnicodeProps.java | 24 +-
.../apache/lucene/analysis/util/package-info.java | 4 +-
.../analysis/wikipedia/WikipediaTokenizer.java | 218 +-
.../wikipedia/WikipediaTokenizerFactory.java | 6 +-
.../analysis/wikipedia/WikipediaTokenizerImpl.java | 1444 ++++----
.../lucene/analysis/wikipedia/package-info.java | 4 +-
.../lucene/analysis/bg/TestBulgarianAnalyzer.java | 37 +-
.../bg/TestBulgarianStemFilterFactory.java | 25 +-
.../lucene/analysis/bg/TestBulgarianStemmer.java | 103 +-
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 12 +-
.../lucene/analysis/bn/TestBengaliFilters.java | 59 +-
.../lucene/analysis/bn/TestBengaliNormalizer.java | 40 +-
.../lucene/analysis/bn/TestBengaliStemmer.java | 31 +-
.../boost/DelimitedBoostTokenFilterTest.java | 22 +-
.../lucene/analysis/br/TestBrazilianAnalyzer.java | 211 +-
.../br/TestBrazilianStemFilterFactory.java | 25 +-
.../lucene/analysis/ca/TestCatalanAnalyzer.java | 21 +-
.../charfilter/HTMLStripCharFilterTest.java | 621 ++--
.../charfilter/TestHTMLStripCharFilterFactory.java | 83 +-
.../analysis/charfilter/TestMappingCharFilter.java | 286 +-
.../charfilter/TestMappingCharFilterFactory.java | 52 +-
.../lucene/analysis/cjk/TestCJKAnalyzer.java | 476 +--
.../lucene/analysis/cjk/TestCJKBigramFilter.java | 287 +-
.../analysis/cjk/TestCJKBigramFilterFactory.java | 51 +-
.../analysis/cjk/TestCJKWidthCharFilter.java | 78 +-
.../cjk/TestCJKWidthCharFilterFactory.java | 20 +-
.../lucene/analysis/cjk/TestCJKWidthFilter.java | 73 +-
.../analysis/cjk/TestCJKWidthFilterFactory.java | 21 +-
.../lucene/analysis/ckb/TestSoraniAnalyzer.java | 32 +-
.../ckb/TestSoraniNormalizationFilter.java | 58 +-
.../ckb/TestSoraniNormalizationFilterFactory.java | 23 +-
.../lucene/analysis/ckb/TestSoraniStemFilter.java | 68 +-
.../analysis/ckb/TestSoraniStemFilterFactory.java | 25 +-
.../analysis/classic/TestClassicAnalyzer.java | 201 +-
.../analysis/classic/TestClassicFactories.java | 54 +-
.../commongrams/CommonGramsFilterTest.java | 390 +--
.../commongrams/TestCommonGramsFilterFactory.java | 81 +-
.../TestCommonGramsQueryFilterFactory.java | 90 +-
.../compound/TestCompoundWordTokenFilter.java | 557 ++--
...stDictionaryCompoundWordTokenFilterFactory.java | 41 +-
...tHyphenationCompoundWordTokenFilterFactory.java | 80 +-
.../core/TestAllAnalyzersHaveFactories.java | 148 +-
.../apache/lucene/analysis/core/TestAnalyzers.java | 162 +-
.../lucene/analysis/core/TestBugInSomething.java | 323 +-
.../lucene/analysis/core/TestCoreFactories.java | 55 +-
.../analysis/core/TestDecimalDigitFilter.java | 121 +-
.../core/TestDecimalDigitFilterFactory.java | 29 +-
.../lucene/analysis/core/TestDuelingAnalyzers.java | 181 +-
.../apache/lucene/analysis/core/TestFactories.java | 73 +-
.../analysis/core/TestFlattenGraphFilter.java | 372 ++-
.../lucene/analysis/core/TestKeywordAnalyzer.java | 26 +-
.../lucene/analysis/core/TestKeywordTokenizer.java | 37 +-
.../lucene/analysis/core/TestRandomChains.java | 776 +++--
.../lucene/analysis/core/TestStopAnalyzer.java | 27 +-
.../analysis/core/TestStopFilterFactory.java | 61 +-
.../lucene/analysis/core/TestTypeTokenFilter.java | 28 +-
.../analysis/core/TestTypeTokenFilterFactory.java | 52 +-
.../core/TestUnicodeWhitespaceTokenizer.java | 54 +-
.../analysis/core/TestWhitespaceAnalyzer.java | 22 +-
.../lucene/analysis/custom/TestCustomAnalyzer.java | 552 ++--
.../lucene/analysis/cz/TestCzechAnalyzer.java | 27 +-
.../analysis/cz/TestCzechStemFilterFactory.java | 27 +-
.../lucene/analysis/cz/TestCzechStemmer.java | 410 ++-
.../lucene/analysis/da/TestDanishAnalyzer.java | 16 +-
.../lucene/analysis/de/TestGermanAnalyzer.java | 25 +-
.../analysis/de/TestGermanLightStemFilter.java | 69 +-
.../de/TestGermanLightStemFilterFactory.java | 21 +-
.../analysis/de/TestGermanMinimalStemFilter.java | 71 +-
.../de/TestGermanMinimalStemFilterFactory.java | 23 +-
.../analysis/de/TestGermanNormalizationFilter.java | 62 +-
.../de/TestGermanNormalizationFilterFactory.java | 23 +-
.../lucene/analysis/de/TestGermanStemFilter.java | 72 +-
.../analysis/de/TestGermanStemFilterFactory.java | 27 +-
.../lucene/analysis/el/GreekAnalyzerTest.java | 45 +-
.../el/TestGreekLowerCaseFilterFactory.java | 25 +-
.../analysis/el/TestGreekStemFilterFactory.java | 21 +-
.../lucene/analysis/el/TestGreekStemmer.java | 186 +-
.../analysis/email/TestUAX29URLEmailAnalyzer.java | 657 ++--
.../analysis/email/TestUAX29URLEmailTokenizer.java | 2426 +++++++++++---
.../email/TestUAX29URLEmailTokenizerFactory.java | 201 +-
.../lucene/analysis/en/TestEnglishAnalyzer.java | 16 +-
.../analysis/en/TestEnglishMinimalStemFilter.java | 46 +-
.../en/TestEnglishMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/en/TestKStemFilterFactory.java | 21 +-
.../apache/lucene/analysis/en/TestKStemmer.java | 113 +-
.../lucene/analysis/en/TestPorterStemFilter.java | 58 +-
.../analysis/en/TestPorterStemFilterFactory.java | 25 +-
.../lucene/analysis/es/TestSpanishAnalyzer.java | 16 +-
.../analysis/es/TestSpanishLightStemFilter.java | 48 +-
.../es/TestSpanishLightStemFilterFactory.java | 23 +-
.../analysis/es/TestSpanishMinimalStemFilter.java | 38 +-
.../es/TestSpanishMinimalStemFilterFactory.java | 20 +-
.../lucene/analysis/et/TestEstonianAnalyzer.java | 69 +-
.../lucene/analysis/eu/TestBasqueAnalyzer.java | 18 +-
.../lucene/analysis/fa/TestPersianAnalyzer.java | 200 +-
.../lucene/analysis/fa/TestPersianCharFilter.java | 34 +-
.../fa/TestPersianNormalizationFilter.java | 30 +-
.../fa/TestPersianNormalizationFilterFactory.java | 25 +-
.../lucene/analysis/fi/TestFinnishAnalyzer.java | 16 +-
.../analysis/fi/TestFinnishLightStemFilter.java | 69 +-
.../fi/TestFinnishLightStemFilterFactory.java | 23 +-
.../lucene/analysis/fr/TestFrenchAnalyzer.java | 142 +-
.../analysis/fr/TestFrenchLightStemFilter.java | 147 +-
.../fr/TestFrenchLightStemFilterFactory.java | 21 +-
.../analysis/fr/TestFrenchMinimalStemFilter.java | 73 +-
.../fr/TestFrenchMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/ga/TestIrishAnalyzer.java | 29 +-
.../analysis/ga/TestIrishLowerCaseFilter.java | 35 +-
.../ga/TestIrishLowerCaseFilterFactory.java | 21 +-
.../lucene/analysis/gl/TestGalicianAnalyzer.java | 16 +-
.../analysis/gl/TestGalicianMinimalStemFilter.java | 69 +-
.../gl/TestGalicianMinimalStemFilterFactory.java | 21 +-
.../lucene/analysis/gl/TestGalicianStemFilter.java | 44 +-
.../analysis/gl/TestGalicianStemFilterFactory.java | 21 +-
.../lucene/analysis/hi/TestHindiAnalyzer.java | 25 +-
.../lucene/analysis/hi/TestHindiFilters.java | 59 +-
.../lucene/analysis/hi/TestHindiNormalizer.java | 31 +-
.../lucene/analysis/hi/TestHindiStemmer.java | 58 +-
.../lucene/analysis/hu/TestHungarianAnalyzer.java | 16 +-
.../analysis/hu/TestHungarianLightStemFilter.java | 67 +-
.../hu/TestHungarianLightStemFilterFactory.java | 21 +-
.../lucene/analysis/hunspell/StemmerTestBase.java | 34 +-
.../lucene/analysis/hunspell/Test64kAffixes.java | 18 +-
.../analysis/hunspell/TestAllDictionaries.java | 308 +-
.../analysis/hunspell/TestAllDictionaries2.java | 424 ++-
.../analysis/hunspell/TestAlternateCasing.java | 35 +-
.../analysis/hunspell/TestCaseInsensitive.java | 5 +-
.../analysis/hunspell/TestCaseSensitive.java | 41 +-
.../lucene/analysis/hunspell/TestCircumfix.java | 5 +-
.../analysis/hunspell/TestComplexPrefix.java | 3 +-
.../lucene/analysis/hunspell/TestCondition.java | 3 +-
.../lucene/analysis/hunspell/TestCondition2.java | 5 +-
.../apache/lucene/analysis/hunspell/TestConv.java | 5 +-
.../lucene/analysis/hunspell/TestDependencies.java | 5 +-
.../lucene/analysis/hunspell/TestDictionary.java | 148 +-
.../lucene/analysis/hunspell/TestDoubleEscape.java | 3 +-
.../lucene/analysis/hunspell/TestEscaped.java | 3 +-
.../lucene/analysis/hunspell/TestFlagLong.java | 3 +-
.../lucene/analysis/hunspell/TestFlagNum.java | 3 +-
.../lucene/analysis/hunspell/TestFullStrip.java | 3 +-
.../lucene/analysis/hunspell/TestHomonyms.java | 5 +-
.../analysis/hunspell/TestHunspellStemFilter.java | 82 +-
.../hunspell/TestHunspellStemFilterFactory.java | 30 +-
.../lucene/analysis/hunspell/TestIgnore.java | 5 +-
.../lucene/analysis/hunspell/TestKeepCase.java | 23 +-
.../apache/lucene/analysis/hunspell/TestMorph.java | 5 +-
.../lucene/analysis/hunspell/TestMorphAlias.java | 9 +-
.../lucene/analysis/hunspell/TestMorphData.java | 9 +-
.../lucene/analysis/hunspell/TestNeedAffix.java | 15 +-
.../analysis/hunspell/TestOnlyInCompound.java | 9 +-
.../analysis/hunspell/TestOptionalCondition.java | 3 +-
.../lucene/analysis/hunspell/TestSpaces.java | 5 +-
.../lucene/analysis/hunspell/TestStemmer.java | 5 +-
.../hunspell/TestStrangeOvergeneration.java | 3 +-
.../lucene/analysis/hunspell/TestTwoFold.java | 5 +-
.../lucene/analysis/hunspell/TestTwoSuffixes.java | 5 +-
.../lucene/analysis/hunspell/TestZeroAffix.java | 3 +-
.../lucene/analysis/hunspell/TestZeroAffix2.java | 5 +-
.../lucene/analysis/hy/TestArmenianAnalyzer.java | 18 +-
.../lucene/analysis/id/TestIndonesianAnalyzer.java | 16 +-
.../id/TestIndonesianStemFilterFactory.java | 37 +-
.../lucene/analysis/id/TestIndonesianStemmer.java | 75 +-
.../lucene/analysis/in/TestIndicNormalizer.java | 34 +-
.../lucene/analysis/it/TestItalianAnalyzer.java | 22 +-
.../analysis/it/TestItalianLightStemFilter.java | 48 +-
.../it/TestItalianLightStemFilterFactory.java | 21 +-
.../lucene/analysis/lt/TestLithuanianAnalyzer.java | 18 +-
.../lucene/analysis/lt/TestLithuanianStemming.java | 773 +++--
.../lucene/analysis/lv/TestLatvianAnalyzer.java | 16 +-
.../analysis/lv/TestLatvianStemFilterFactory.java | 23 +-
.../lucene/analysis/lv/TestLatvianStemmer.java | 330 +-
.../lucene/analysis/minhash/MinHashFilterTest.java | 133 +-
.../DateRecognizerFilterFactoryTest.java | 20 +-
.../miscellaneous/DateRecognizerFilterTest.java | 9 +-
.../DelimitedTermFrequencyTokenFilterTest.java | 11 +-
.../miscellaneous/TestASCIIFoldingFilter.java | 3475 ++++++++++----------
.../TestAsciiFoldingFilterFactory.java | 18 +-
.../miscellaneous/TestCapitalizationFilter.java | 366 ++-
.../TestCapitalizationFilterFactory.java | 425 ++-
.../miscellaneous/TestCodepointCountFilter.java | 37 +-
.../TestCodepointCountFilterFactory.java | 55 +-
.../miscellaneous/TestConcatenateGraphFilter.java | 72 +-
.../TestConcatenateGraphFilterFactory.java | 64 +-
.../TestConcatenatingTokenStream.java | 73 +-
.../miscellaneous/TestConditionalTokenFilter.java | 276 +-
.../miscellaneous/TestDropIfFlaggedFilter.java | 39 +-
.../TestDropIfFlaggedFilterFactory.java | 18 +-
.../miscellaneous/TestEmptyTokenStream.java | 11 +-
.../miscellaneous/TestFingerprintFilter.java | 26 +-
.../TestFingerprintFilterFactory.java | 42 +-
.../miscellaneous/TestFixBrokenOffsetsFilter.java | 7 +-
.../miscellaneous/TestHyphenatedWordsFilter.java | 71 +-
.../miscellaneous/TestKeepFilterFactory.java | 26 +-
.../analysis/miscellaneous/TestKeepWordFilter.java | 49 +-
.../miscellaneous/TestKeywordMarkerFilter.java | 122 +-
.../TestKeywordMarkerFilterFactory.java | 108 +-
.../miscellaneous/TestKeywordRepeatFilter.java | 33 +-
.../analysis/miscellaneous/TestLengthFilter.java | 31 +-
.../miscellaneous/TestLengthFilterFactory.java | 58 +-
.../miscellaneous/TestLimitTokenCountAnalyzer.java | 55 +-
.../miscellaneous/TestLimitTokenCountFilter.java | 5 +-
.../TestLimitTokenCountFilterFactory.java | 56 +-
.../miscellaneous/TestLimitTokenOffsetFilter.java | 7 +-
.../TestLimitTokenOffsetFilterFactory.java | 54 +-
.../TestLimitTokenPositionFilter.java | 91 +-
.../TestLimitTokenPositionFilterFactory.java | 75 +-
.../miscellaneous/TestMiscellaneousFactories.java | 29 +-
.../miscellaneous/TestPerFieldAnalyzerWrapper.java | 108 +-
.../miscellaneous/TestProtectedTermFilter.java | 17 +-
.../TestProtectedTermFilterFactory.java | 207 +-
.../TestRemoveDuplicatesTokenFilter.java | 191 +-
.../TestRemoveDuplicatesTokenFilterFactory.java | 36 +-
.../TestScandinavianFoldingFilter.java | 45 +-
.../TestScandinavianFoldingFilterFactory.java | 19 +-
.../TestScandinavianNormalizationFilter.java | 46 +-
...TestScandinavianNormalizationFilterFactory.java | 16 +-
.../miscellaneous/TestStemmerOverrideFilter.java | 62 +-
.../TestStemmerOverrideFilterFactory.java | 52 +-
.../analysis/miscellaneous/TestTrimFilter.java | 52 +-
.../miscellaneous/TestTrimFilterFactory.java | 21 +-
.../miscellaneous/TestTruncateTokenFilter.java | 8 +-
.../TestTruncateTokenFilterFactory.java | 68 +-
.../miscellaneous/TestTypeAsSynonymFilter.java | 87 +-
.../TestTypeAsSynonymFilterFactory.java | 37 +-
.../miscellaneous/TestWordDelimiterFilter.java | 651 ++--
.../TestWordDelimiterGraphFilter.java | 1062 +++---
.../analysis/ngram/EdgeNGramTokenFilterTest.java | 160 +-
.../analysis/ngram/EdgeNGramTokenizerTest.java | 116 +-
.../analysis/ngram/NGramTokenFilterTest.java | 224 +-
.../lucene/analysis/ngram/NGramTokenizerTest.java | 131 +-
.../lucene/analysis/ngram/TestNGramFilters.java | 180 +-
.../lucene/analysis/nl/TestDutchAnalyzer.java | 210 +-
.../lucene/analysis/no/TestNorwegianAnalyzer.java | 16 +-
.../analysis/no/TestNorwegianLightStemFilter.java | 90 +-
.../no/TestNorwegianLightStemFilterFactory.java | 23 +-
.../no/TestNorwegianMinimalStemFilter.java | 92 +-
.../no/TestNorwegianMinimalStemFilterFactory.java | 35 +-
.../analysis/path/TestPathHierarchyTokenizer.java | 264 +-
.../path/TestReversePathHierarchyTokenizer.java | 219 +-
.../TestPatternCaptureGroupTokenFilter.java | 554 ++--
.../pattern/TestPatternReplaceCharFilter.java | 242 +-
.../TestPatternReplaceCharFilterFactory.java | 84 +-
.../analysis/pattern/TestPatternReplaceFilter.java | 115 +-
.../pattern/TestPatternReplaceFilterFactory.java | 34 +-
.../analysis/pattern/TestPatternTokenizer.java | 125 +-
.../pattern/TestPatternTokenizerFactory.java | 23 +-
.../pattern/TestSimplePatternSplitTokenizer.java | 135 +-
.../pattern/TestSimplePatternTokenizer.java | 97 +-
.../payloads/DelimitedPayloadTokenFilterTest.java | 52 +-
.../payloads/NumericPayloadTokenFilterTest.java | 33 +-
.../TestDelimitedPayloadTokenFilterFactory.java | 27 +-
.../TokenOffsetPayloadTokenFilterTest.java | 16 +-
.../payloads/TypeAsPayloadTokenFilterTest.java | 20 +-
.../lucene/analysis/pt/TestPortugueseAnalyzer.java | 16 +-
.../analysis/pt/TestPortugueseLightStemFilter.java | 166 +-
.../pt/TestPortugueseLightStemFilterFactory.java | 23 +-
.../pt/TestPortugueseMinimalStemFilter.java | 120 +-
.../pt/TestPortugueseMinimalStemFilterFactory.java | 23 +-
.../analysis/pt/TestPortugueseStemFilter.java | 120 +-
.../pt/TestPortugueseStemFilterFactory.java | 21 +-
.../query/QueryAutoStopWordAnalyzerTest.java | 65 +-
.../analysis/reverse/TestReverseStringFilter.java | 67 +-
.../reverse/TestReverseStringFilterFactory.java | 25 +-
.../lucene/analysis/ro/TestRomanianAnalyzer.java | 16 +-
.../lucene/analysis/ru/TestRussianAnalyzer.java | 41 +-
.../analysis/ru/TestRussianLightStemFilter.java | 69 +-
.../ru/TestRussianLightStemFilterFactory.java | 21 +-
.../analysis/shingle/FixedShingleFilterTest.java | 362 +-
.../shingle/ShingleAnalyzerWrapperTest.java | 551 ++--
.../lucene/analysis/shingle/ShingleFilterTest.java | 2240 +++++++------
.../analysis/shingle/TestShingleFilterFactory.java | 238 +-
.../analysis/sinks/TestTeeSinkTokenFilter.java | 74 +-
.../lucene/analysis/snowball/TestSnowball.java | 86 +-
.../snowball/TestSnowballPorterFilterFactory.java | 43 +-
.../analysis/snowball/TestSnowballVocab.java | 37 +-
.../lucene/analysis/sr/TestSerbianAnalyzer.java | 16 +-
.../sr/TestSerbianNormalizationFilter.java | 57 +-
.../sr/TestSerbianNormalizationFilterFactory.java | 28 +-
.../sr/TestSerbianNormalizationRegularFilter.java | 58 +-
.../lucene/analysis/sv/TestSwedishAnalyzer.java | 16 +-
.../analysis/sv/TestSwedishLightStemFilter.java | 69 +-
.../sv/TestSwedishLightStemFilterFactory.java | 21 +-
.../synonym/BaseSynonymParserTestCase.java | 61 +-
.../analysis/synonym/TestMultiWordSynonyms.java | 27 +-
.../analysis/synonym/TestSolrSynonymParser.java | 232 +-
.../analysis/synonym/TestSynonymFilterFactory.java | 148 +-
.../analysis/synonym/TestSynonymGraphFilter.java | 1330 ++++----
.../analysis/synonym/TestSynonymMapFilter.java | 628 ++--
.../analysis/synonym/TestWordnetSynonymParser.java | 70 +-
.../lucene/analysis/th/TestThaiAnalyzer.java | 110 +-
.../analysis/th/TestThaiTokenizerFactory.java | 33 +-
.../lucene/analysis/tr/TestApostropheFilter.java | 4 +-
.../analysis/tr/TestApostropheFilterFactory.java | 32 +-
.../lucene/analysis/tr/TestTurkishAnalyzer.java | 13 +-
.../analysis/tr/TestTurkishLowerCaseFilter.java | 72 +-
.../tr/TestTurkishLowerCaseFilterFactory.java | 27 +-
.../analysis/util/StringMockResourceLoader.java | 6 +-
.../analysis/util/TestCharArrayIterator.java | 46 +-
.../lucene/analysis/util/TestCharTokenizers.java | 171 +-
.../apache/lucene/analysis/util/TestElision.java | 25 +-
.../analysis/util/TestElisionFilterFactory.java | 51 +-
.../util/TestFilesystemResourceLoader.java | 61 +-
.../analysis/util/TestRollingCharBuffer.java | 17 +-
.../analysis/util/TestSegmentingTokenizerBase.java | 177 +-
.../wikipedia/TestWikipediaTokenizerFactory.java | 173 +-
.../analysis/wikipedia/WikipediaTokenizerTest.java | 521 ++-
.../analysis/standard/GenerateJflexTLDMacros.java | 177 +-
.../apache/lucene/analysis/LowerCaseFilter.java | 4 +-
.../tokenattributes/PackedTokenAttributeImpl.java | 4 +-
.../org/apache/lucene/search/PhrasePositions.java | 4 +-
.../org/apache/lucene/util/AttributeSource.java | 4 +-
.../java/org/apache/lucene/util/UnicodeUtil.java | 8 +-
.../apache/lucene/util/automaton/StatePair.java | 4 +-
.../apache/lucene/queries/CommonTermsQuery.java | 249 +-
.../queries/function/FunctionMatchQuery.java | 51 +-
.../lucene/queries/function/FunctionQuery.java | 42 +-
.../queries/function/FunctionRangeQuery.java | 79 +-
.../queries/function/FunctionScoreQuery.java | 140 +-
.../lucene/queries/function/FunctionValues.java | 154 +-
.../queries/function/IndexReaderFunctions.java | 27 +-
.../lucene/queries/function/ValueSource.java | 88 +-
.../lucene/queries/function/ValueSourceScorer.java | 55 +-
.../queries/function/docvalues/BoolDocValues.java | 11 +-
.../function/docvalues/DocTermsIndexDocValues.java | 37 +-
.../function/docvalues/DoubleDocValues.java | 40 +-
.../queries/function/docvalues/FloatDocValues.java | 11 +-
.../queries/function/docvalues/IntDocValues.java | 27 +-
.../queries/function/docvalues/LongDocValues.java | 29 +-
.../queries/function/docvalues/StrDocValues.java | 1 -
.../queries/function/docvalues/package-info.java | 4 +-
.../lucene/queries/function/package-info.java | 4 +-
.../queries/function/valuesource/BoolFunction.java | 4 +-
.../function/valuesource/BytesRefFieldSource.java | 16 +-
.../valuesource/ComparisonBoolFunction.java | 22 +-
.../function/valuesource/ConstNumberSource.java | 13 +-
.../function/valuesource/ConstValueSource.java | 30 +-
.../queries/function/valuesource/DefFunction.java | 23 +-
.../function/valuesource/DivFloatFunction.java | 12 +-
.../function/valuesource/DocFreqValueSource.java | 38 +-
.../valuesource/DoubleConstValueSource.java | 20 +-
.../function/valuesource/DoubleFieldSource.java | 20 +-
.../function/valuesource/DualFloatFunction.java | 44 +-
.../function/valuesource/EnumFieldSource.java | 54 +-
.../function/valuesource/FieldCacheSource.java | 11 +-
.../function/valuesource/FloatFieldSource.java | 29 +-
.../function/valuesource/IDFValueSource.java | 35 +-
.../queries/function/valuesource/IfFunction.java | 37 +-
.../function/valuesource/IntFieldSource.java | 24 +-
.../valuesource/JoinDocFreqValueSource.java | 24 +-
.../function/valuesource/LinearFloatFunction.java | 27 +-
.../function/valuesource/LiteralValueSource.java | 18 +-
.../function/valuesource/LongFieldSource.java | 25 +-
.../function/valuesource/MaxDocValueSource.java | 17 +-
.../function/valuesource/MaxFloatFunction.java | 15 +-
.../function/valuesource/MinFloatFunction.java | 18 +-
.../function/valuesource/MultiBoolFunction.java | 16 +-
.../function/valuesource/MultiFloatFunction.java | 45 +-
.../function/valuesource/MultiFunction.java | 76 +-
.../function/valuesource/MultiValueSource.java | 8 +-
.../valuesource/MultiValuedDoubleFieldSource.java | 24 +-
.../valuesource/MultiValuedFloatFieldSource.java | 24 +-
.../valuesource/MultiValuedIntFieldSource.java | 24 +-
.../valuesource/MultiValuedLongFieldSource.java | 24 +-
.../function/valuesource/NormValueSource.java | 47 +-
.../function/valuesource/NumDocsValueSource.java | 18 +-
.../function/valuesource/PowFloatFunction.java | 16 +-
.../function/valuesource/ProductFloatFunction.java | 5 +-
.../function/valuesource/QueryValueSource.java | 52 +-
.../valuesource/RangeMapFloatFunction.java | 75 +-
.../valuesource/ReciprocalFloatFunction.java | 61 +-
.../function/valuesource/ScaleFloatFunction.java | 75 +-
.../function/valuesource/SimpleBoolFunction.java | 21 +-
.../function/valuesource/SimpleFloatFunction.java | 16 +-
.../function/valuesource/SingleFunction.java | 17 +-
.../function/valuesource/SortedSetFieldSource.java | 22 +-
.../function/valuesource/SumFloatFunction.java | 7 +-
.../valuesource/SumTotalTermFreqValueSource.java | 31 +-
.../function/valuesource/TFValueSource.java | 131 +-
.../function/valuesource/TermFreqValueSource.java | 119 +-
.../valuesource/TotalTermFreqValueSource.java | 39 +-
.../function/valuesource/VectorValueSource.java | 35 +-
.../queries/function/valuesource/package-info.java | 4 +-
.../queries/intervals/BlockIntervalsSource.java | 14 +-
.../queries/intervals/CachingMatchesIterator.java | 17 +-
.../lucene/queries/intervals/ConjunctionDISI.java | 16 +-
.../intervals/ConjunctionIntervalIterator.java | 2 -
.../intervals/ConjunctionIntervalsSource.java | 21 +-
.../intervals/ContainedByIntervalsSource.java | 6 +-
.../intervals/ContainingIntervalsSource.java | 6 +-
.../intervals/DifferenceIntervalsSource.java | 16 +-
.../queries/intervals/DisiPriorityQueue.java | 15 +-
.../lucene/queries/intervals/DisiWrapper.java | 1 -
.../intervals/DisjunctionDISIApproximation.java | 8 +-
.../intervals/DisjunctionIntervalsSource.java | 195 +-
.../lucene/queries/intervals/Disjunctions.java | 15 +-
.../intervals/ExtendedIntervalIterator.java | 15 +-
.../queries/intervals/ExtendedIntervalsSource.java | 15 +-
.../queries/intervals/FilteredIntervalsSource.java | 38 +-
.../intervals/FixedFieldIntervalsSource.java | 11 +-
.../lucene/queries/intervals/IntervalFilter.java | 15 +-
.../lucene/queries/intervals/IntervalIterator.java | 54 +-
.../lucene/queries/intervals/IntervalMatches.java | 11 +-
.../queries/intervals/IntervalMatchesIterator.java | 10 +-
.../lucene/queries/intervals/IntervalQuery.java | 102 +-
.../queries/intervals/IntervalScoreFunction.java | 27 +-
.../lucene/queries/intervals/IntervalScorer.java | 12 +-
.../apache/lucene/queries/intervals/Intervals.java | 222 +-
.../lucene/queries/intervals/IntervalsSource.java | 43 +-
.../MinimizingConjunctionMatchesIterator.java | 6 +-
.../MinimumShouldMatchIntervalsSource.java | 54 +-
.../intervals/MultiTermIntervalsSource.java | 28 +-
.../intervals/NonOverlappingIntervalsSource.java | 6 +-
.../intervals/NotContainedByIntervalsSource.java | 12 +-
.../intervals/NotContainingIntervalsSource.java | 10 +-
.../queries/intervals/OffsetIntervalsSource.java | 19 +-
.../queries/intervals/OrderedIntervalsSource.java | 32 +-
.../intervals/OverlappingIntervalsSource.java | 10 +-
.../PayloadFilteredTermIntervalsSource.java | 43 +-
.../lucene/queries/intervals/RelativeIterator.java | 3 +-
.../intervals/RepeatingIntervalsSource.java | 24 +-
.../queries/intervals/TermIntervalsSource.java | 64 +-
.../intervals/UnorderedIntervalsSource.java | 30 +-
.../lucene/queries/intervals/package-info.java | 65 +-
.../apache/lucene/queries/mlt/MoreLikeThis.java | 366 +--
.../lucene/queries/mlt/MoreLikeThisQuery.java | 42 +-
.../apache/lucene/queries/mlt/package-info.java | 4 +-
.../org/apache/lucene/queries/package-info.java | 4 +-
.../queries/payloads/AveragePayloadFunction.java | 25 +-
.../queries/payloads/MaxPayloadFunction.java | 25 +-
.../queries/payloads/MinPayloadFunction.java | 26 +-
.../lucene/queries/payloads/PayloadDecoder.java | 13 +-
.../lucene/queries/payloads/PayloadFunction.java | 37 +-
.../lucene/queries/payloads/PayloadScoreQuery.java | 86 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 67 +-
.../queries/payloads/SumPayloadFunction.java | 23 +-
.../lucene/queries/payloads/package-info.java | 16 +-
.../lucene/queries/TestCommonTermsQuery.java | 308 +-
.../lucene/queries/function/FunctionTestSetup.java | 101 +-
.../function/TestDocValuesFieldSources.java | 18 +-
.../queries/function/TestFieldScoreQuery.java | 72 +-
.../queries/function/TestFunctionMatchQuery.java | 12 +-
.../function/TestFunctionQueryExplanations.java | 16 +-
.../queries/function/TestFunctionQuerySort.java | 18 +-
.../queries/function/TestFunctionRangeQuery.java | 44 +-
.../function/TestFunctionScoreExplanations.java | 11 +-
.../queries/function/TestFunctionScoreQuery.java | 170 +-
.../queries/function/TestIndexReaderFunctions.java | 80 +-
.../queries/function/TestLongNormValueSource.java | 6 +-
.../queries/function/TestSortedSetFieldSource.java | 14 +-
.../lucene/queries/function/TestValueSources.java | 562 ++--
.../docvalues/TestBoolValOfNumericDVs.java | 72 +-
.../queries/intervals/OneTimeIntervalSource.java | 14 +-
.../queries/intervals/TestDisjunctionRewrites.java | 213 +-
.../queries/intervals/TestIntervalQuery.java | 361 +-
.../lucene/queries/intervals/TestIntervals.java | 869 +++--
.../intervals/TestPayloadFilteredInterval.java | 33 +-
.../queries/intervals/TestSimplifications.java | 44 +-
.../lucene/queries/mlt/TestMoreLikeThis.java | 280 +-
.../lucene/queries/payloads/PayloadHelper.java | 54 +-
.../queries/payloads/TestPayloadCheckQuery.java | 104 +-
.../queries/payloads/TestPayloadExplanations.java | 69 +-
.../queries/payloads/TestPayloadScoreQuery.java | 217 +-
.../lucene/queries/payloads/TestPayloadSpans.java | 226 +-
.../queries/payloads/TestPayloadTermQuery.java | 152 +-
.../org/apache/lucene/spatial/SpatialTestCase.java | 4 -
.../solr/handler/FieldAnalysisRequestHandler.java | 2 -
896 files changed, 44593 insertions(+), 39638 deletions(-)
diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle
index f6e8351..52bf08d 100644
--- a/gradle/generation/jflex.gradle
+++ b/gradle/generation/jflex.gradle
@@ -26,14 +26,6 @@ configure(rootProject) {
dependencies {
jflex "de.jflex:jflex:${scriptDepVersions['jflex']}"
}
-
- task jflex() {
- description "Regenerate sources for corresponding jflex grammar files."
- group "generation"
-
- dependsOn ":lucene:core:jflexStandardTokenizerImpl"
- dependsOn ":lucene:analysis:common:jflexUAX29URLEmailTokenizerImpl"
- }
}
// We always regenerate, no need to declare outputs.
@@ -115,6 +107,22 @@ configure(project(":lucene:core")) {
configure(project(":lucene:analysis:common")) {
+ task jflexWikipediaTokenizerImpl(type: JFlexTask) {
+ description "Regenerate WikipediaTokenizerImpl.java"
+ group "generation"
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex')
+ skeleton = project(":lucene:core").file("src/data/jflex/skeleton.default")
+ }
+
+ task jflexClassicTokenizerImpl(type: JFlexTask) {
+ description "Regenerate ClassicTokenizerImpl.java"
+ group "generation"
+
+ jflexFile = file('src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex')
+ skeleton = project(":lucene:core").file("src/data/jflex/skeleton.default")
+ }
+
task jflexUAX29URLEmailTokenizerImpl(type: JFlexTask) {
description "Regenerate UAX29URLEmailTokenizerImpl.java"
group "generation"
@@ -166,4 +174,26 @@ configure(project(":lucene:analysis:common")) {
)
}
}
+
+ task regenerate() {
+ description "Regenerate any generated sources"
+ group "generation"
+
+ // Run regeneration tasks.
+ dependsOn jflexUAX29URLEmailTokenizerImpl
+ dependsOn jflexHTMLStripCharFilter
+ dependsOn jflexClassicTokenizerImpl
+ dependsOn jflexWikipediaTokenizerImpl
+
+ // Clean up and reformat the generated sources after generation.
+ dependsOn "tidy"
+ }
+
+ // Make sure tidy runs after generation, if they're defined.
+ tasks.matching { it.name == "tidy" }.configureEach {
+ mustRunAfter jflexUAX29URLEmailTokenizerImpl,
+ jflexHTMLStripCharFilter,
+ jflexClassicTokenizerImpl,
+ jflexWikipediaTokenizerImpl
+ }
}
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle
index 7276c9d..492f306 100644
--- a/gradle/validation/spotless.gradle
+++ b/gradle/validation/spotless.gradle
@@ -29,42 +29,42 @@ allprojects { prj ->
java {
// TODO: work out how to have multiple different header files (we have
// classes in the codebase that have original headers).
-
// licenseHeaderFile file("${resources}/asl-header.txt"), '^(\\s*package)'
+
lineEndings 'UNIX'
endWithNewline()
googleJavaFormat('1.9')
switch (project.path) {
// These modules are complete - all sources scanned.
+ case ":lucene:core":
+ target "src/java/**/*.java",
+ "src/test/**/*.java"
+ targetExclude "**/resources/**", "**/StandardTokenizerImpl.java"
+ break
+
case ":lucene:highlighter":
- target "src/java/**", "src/test/**"
- targetExclude "**/overview.html", "**/CambridgeMA.utf8"
+ target "src/java/**/*.java",
+ "src/test/**/*.java"
+ targetExclude "**/resources/**"
break
- // Partially complete.
- case ":lucene:core":
- target "src/java/**", "src/test/**"
- targetExclude "**/overview.html",
- "**/META-INF/**",
- "**/StandardTokenizerImpl.jflex",
- "**/StandardTokenizerImpl.java",
- "**/createLevAutomata.py",
- "**/UTF32ToUTF8.py",
- "**/gen_BulkOperation.py",
- "**/gen_Packed64SingleBlock.py",
- "**/makeEuroparlLineFile.py",
- "**/wordliststopwords.txt",
- "**/wordliststopwords_nocomment.txt",
- "**/gen_ForUtil.py"
+ case ":lucene:queries":
+ target "src/java/**/*.java",
+ "src/test/**/*.java"
+ targetExclude "**/resources/**"
break
case ":lucene:analysis:common":
- target "src/**/org/apache/lucene/analysis/ar/**",
- "src/**/org/apache/lucene/collation/**"
- targetExclude "**/resources/**"
+ target "src/**/*.java"
+ targetExclude "**/resources/**",
+ "**/HTMLStripCharFilter.java",
+ "**/UAX29URLEmailTokenizerImpl.java",
+ "**/tartarus/**"
break
+ // Partially complete.
+
// All others - disable reformatting/ checks for now.
case ":lucene:analysis:icu":
case ":lucene:analysis:kuromoji":
@@ -87,7 +87,6 @@ allprojects { prj ->
case ":lucene:memory":
case ":lucene:misc":
case ":lucene:monitor":
- case ":lucene:queries":
case ":lucene:queryparser":
case ":lucene:replicator":
case ":lucene:sandbox":
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
index 7077d2b..2e50563 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.bg;
-
import java.io.IOException;
import java.io.Reader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -32,10 +30,9 @@ import org.apache.lucene.analysis.standard.StandardTokenizer;
/**
* {@link Analyzer} for Bulgarian.
- * <p>
- * This analyzer implements light-stemming as specified by: <i> Searching
- * Strategies for the Bulgarian Language </i>
- * http://members.unine.ch/jacques.savoy/Papers/BUIR.pdf
+ *
+ * <p>This analyzer implements light-stemming as specified by: <i> Searching Strategies for the
+ * Bulgarian Language </i> http://members.unine.ch/jacques.savoy/Papers/BUIR.pdf
*
* @since 3.1
*/
@@ -43,32 +40,32 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default Bulgarian stopwords.
- *
- * Default stopword list is from
- * http://members.unine.ch/jacques.savoy/clef/index.html The stopword list is
- * BSD-Licensed.
+ *
+ * <p>Default stopword list is from http://members.unine.ch/jacques.savoy/clef/index.html The
+ * stopword list is BSD-Licensed.
*/
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
/**
* Returns an unmodifiable instance of the default stop-words set.
- *
+ *
* @return an unmodifiable instance of the default stop-words set.
*/
public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
/**
- * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer
- * class accesses the static final set the first time.;
+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the
+ * static final set the first time.;
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
-
+
static {
try {
- DEFAULT_STOP_SET = loadStopwordSet(false, BulgarianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
+ DEFAULT_STOP_SET =
+ loadStopwordSet(false, BulgarianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@@ -76,53 +73,46 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
}
}
}
-
+
private final CharArraySet stemExclusionSet;
-
- /**
- * Builds an analyzer with the default stop words:
- * {@link #DEFAULT_STOPWORD_FILE}.
- */
+
+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
public BulgarianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
-
- /**
- * Builds an analyzer with the given stop words.
- */
+
+ /** Builds an analyzer with the given stop words. */
public BulgarianAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
-
+
/**
- * Builds an analyzer with the given stop words and a stem exclusion set.
- * If a stem exclusion set is provided this analyzer will add a {@link SetKeywordMarkerFilter}
- * before {@link BulgarianStemFilter}.
+ * Builds an analyzer with the given stop words and a stem exclusion set. If a stem exclusion set
+ * is provided this analyzer will add a {@link SetKeywordMarkerFilter} before {@link
+ * BulgarianStemFilter}.
*/
public BulgarianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
super(stopwords);
- this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
+ this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
/**
- * Creates a
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * which tokenizes all the text in the provided {@link Reader}.
- *
- * @return A
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from an {@link StandardTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link StopFilter}
- * , {@link SetKeywordMarkerFilter} if a stem exclusion set is
- * provided and {@link BulgarianStemFilter}.
+ * Creates a {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} which tokenizes all
+ * the text in the provided {@link Reader}.
+ *
+ * @return A {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from an
+ * {@link StandardTokenizer} filtered with {@link LowerCaseFilter}, {@link StopFilter}, {@link
+ * SetKeywordMarkerFilter} if a stem exclusion set is provided and {@link
+ * BulgarianStemFilter}.
*/
@Override
public TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new StopFilter(result, stopwords);
- if(!stemExclusionSet.isEmpty())
+ if (!stemExclusionSet.isEmpty()) {
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+ }
result = new BulgarianStemFilter(result);
return new TokenStreamComponents(source, result);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
index eec74a6..9618ea6 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
@@ -16,37 +16,33 @@
*/
package org.apache.lucene.analysis.bg;
-
import java.io.IOException;
-
-import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // for javadoc
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // for javadoc
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
/**
- * A {@link TokenFilter} that applies {@link BulgarianStemmer} to stem Bulgarian
- * words.
- * <p>
- * To prevent terms from being stemmed use an instance of
- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
- * the {@link KeywordAttribute} before this {@link TokenStream}.
- * </p>
+ * A {@link TokenFilter} that applies {@link BulgarianStemmer} to stem Bulgarian words.
+ *
+ * <p>To prevent terms from being stemmed use an instance of {@link SetKeywordMarkerFilter} or a
+ * custom {@link TokenFilter} that sets the {@link KeywordAttribute} before this {@link
+ * TokenStream}.
*/
public final class BulgarianStemFilter extends TokenFilter {
private final BulgarianStemmer stemmer = new BulgarianStemmer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
-
+
public BulgarianStemFilter(final TokenStream input) {
super(input);
}
-
+
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- if(!keywordAttr.isKeyword()) {
+ if (!keywordAttr.isKeyword()) {
final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilterFactory.java
index 29fee9f..8aebc1e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.bg;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
+/**
* Factory for {@link BulgarianStemFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_bgstem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -32,6 +31,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.BulgarianStemFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 3.1.0
* @lucene.spi {@value #NAME}
*/
@@ -41,13 +41,13 @@ public class BulgarianStemFilterFactory extends TokenFilterFactory {
public static final String NAME = "bulgarianStem";
/** Creates a new BulgarianStemFilterFactory */
- public BulgarianStemFilterFactory(Map<String,String> args) {
+ public BulgarianStemFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public BulgarianStemFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemmer.java
index 0f5ebb2..5f727f8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemmer.java
@@ -16,46 +16,43 @@
*/
package org.apache.lucene.analysis.bg;
-
import static org.apache.lucene.analysis.util.StemmerUtil.*;
/**
* Light Stemmer for Bulgarian.
- * <p>
- * Implements the algorithm described in:
- * <i>
- * Searching Strategies for the Bulgarian Language
- * </i>
- * http://members.unine.ch/jacques.savoy/Papers/BUIR.pdf
+ *
+ * <p>Implements the algorithm described in: <i> Searching Strategies for the Bulgarian Language
+ * </i> http://members.unine.ch/jacques.savoy/Papers/BUIR.pdf
*/
public class BulgarianStemmer {
-
+
/**
* Stem an input buffer of Bulgarian text.
- *
+ *
* @param s input buffer
* @param len length of input buffer
* @return length of input buffer after normalization
*/
public int stem(final char s[], int len) {
if (len < 4) // do not stem
- return len;
-
- if (len > 5 && endsWith(s, len, "ища"))
+ return len;
+
+ if (len > 5 && endsWith(s, len, "ища")) {
return len - 3;
-
+ }
+
len = removeArticle(s, len);
len = removePlural(s, len);
-
+
if (len > 3) {
- if (endsWith(s, len, "я"))
+ if (endsWith(s, len, "я")) {
len--;
- if (endsWith(s, len, "а") ||
- endsWith(s, len, "о") ||
- endsWith(s, len, "е"))
+ }
+ if (endsWith(s, len, "а") || endsWith(s, len, "о") || endsWith(s, len, "е")) {
len--;
+ }
}
-
+
// the rule to rewrite ен -> н is duplicated in the paper.
// in the perl implementation referenced by the paper, this is fixed.
// (it is fixed here as well)
@@ -63,7 +60,7 @@ public class BulgarianStemmer {
s[len - 2] = 'н'; // replace with н
len--;
}
-
+
if (len > 5 && s[len - 2] == 'ъ') {
s[len - 2] = s[len - 1]; // replace ъN with N
len--;
@@ -71,49 +68,47 @@ public class BulgarianStemmer {
return len;
}
-
+
/**
* Mainly remove the definite article
+ *
* @param s input buffer
* @param len length of input buffer
* @return new stemmed length
*/
private int removeArticle(final char s[], final int len) {
- if (len > 6 && endsWith(s, len, "ият"))
- return len - 3;
-
+ if (len > 6 && endsWith(s, len, "ият")) return len - 3;
+
if (len > 5) {
- if (endsWith(s, len, "ът") ||
- endsWith(s, len, "то") ||
- endsWith(s, len, "те") ||
- endsWith(s, len, "та") ||
- endsWith(s, len, "ия"))
+ if (endsWith(s, len, "ът")
+ || endsWith(s, len, "то")
+ || endsWith(s, len, "те")
+ || endsWith(s, len, "та")
+ || endsWith(s, len, "ия")) {
return len - 2;
+ }
}
-
- if (len > 4 && endsWith(s, len, "ят"))
+
+ if (len > 4 && endsWith(s, len, "ят")) {
return len - 2;
+ }
return len;
}
-
+
private int removePlural(final char s[], final int len) {
if (len > 6) {
- if (endsWith(s, len, "овци"))
- return len - 3; // replace with о
- if (endsWith(s, len, "ове"))
- return len - 3;
+ if (endsWith(s, len, "овци")) return len - 3; // replace with о
+ if (endsWith(s, len, "ове")) return len - 3;
if (endsWith(s, len, "еве")) {
s[len - 3] = 'й'; // replace with й
return len - 2;
}
}
-
+
if (len > 5) {
- if (endsWith(s, len, "ища"))
- return len - 3;
- if (endsWith(s, len, "та"))
- return len - 2;
+ if (endsWith(s, len, "ища")) return len - 3;
+ if (endsWith(s, len, "та")) return len - 2;
if (endsWith(s, len, "ци")) {
s[len - 2] = 'к'; // replace with к
return len - 1;
@@ -122,22 +117,21 @@ public class BulgarianStemmer {
s[len - 2] = 'г'; // replace with г
return len - 1;
}
-
+
if (s[len - 3] == 'е' && s[len - 1] == 'и') {
s[len - 3] = 'я'; // replace е with я, remove и
return len - 1;
}
}
-
+
if (len > 4) {
if (endsWith(s, len, "си")) {
s[len - 2] = 'х'; // replace with х
return len - 1;
}
- if (endsWith(s, len, "и"))
- return len - 1;
+ if (endsWith(s, len, "и")) return len - 1;
}
-
+
return len;
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/package-info.java
index c60a53b..d4aaeaf 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analyzer for Bulgarian.
- */
+/** Analyzer for Bulgarian. */
package org.apache.lucene.analysis.bg;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java
index fe4d08a..dc21f14 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.bn;
-
import java.io.IOException;
import java.io.Reader;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
@@ -38,43 +36,46 @@ import org.apache.lucene.analysis.standard.StandardTokenizer;
*/
public final class BengaliAnalyzer extends StopwordAnalyzerBase {
private final CharArraySet stemExclusionSet;
-
+
/**
* File containing default Bengali stopwords.
- *
- * Default stopword list is from http://members.unine.ch/jacques.savoy/clef/bengaliST.txt
- * The stopword list is BSD-Licensed.
+ *
+ * <p>Default stopword list is from http://members.unine.ch/jacques.savoy/clef/bengaliST.txt The
+ * stopword list is BSD-Licensed.
*/
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
private static final String STOPWORDS_COMMENT = "#";
-
+
/**
* Returns an unmodifiable instance of the default stop-words set.
+ *
* @return an unmodifiable instance of the default stop-words set.
*/
- public static CharArraySet getDefaultStopSet(){
+ public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
/**
- * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
- * accesses the static final set the first time.;
+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the
+ * static final set the first time.;
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
static {
try {
- DEFAULT_STOP_SET = loadStopwordSet(false, BengaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
+ DEFAULT_STOP_SET =
+ loadStopwordSet(false, BengaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
} catch (IOException ex) {
throw new RuntimeException("Unable to load default stopword set");
}
}
}
-
+
/**
* Builds an analyzer with the given stop words
- *
+ *
* @param stopwords a stopword set
* @param stemExclusionSet a stemming exclusion set
*/
@@ -82,43 +83,39 @@ public final class BengaliAnalyzer extends StopwordAnalyzerBase {
super(stopwords);
this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
}
-
+
/**
- * Builds an analyzer with the given stop words
- *
+ * Builds an analyzer with the given stop words
+ *
* @param stopwords a stopword set
*/
public BengaliAnalyzer(CharArraySet stopwords) {
this(stopwords, CharArraySet.EMPTY_SET);
}
-
- /**
- * Builds an analyzer with the default stop words:
- * {@link #DEFAULT_STOPWORD_FILE}.
- */
+
+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
public BengaliAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
- * Creates
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * used to tokenize all the text in the provided {@link Reader}.
- *
- * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from a {@link StandardTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link IndicNormalizationFilter},
- * {@link BengaliNormalizationFilter}, {@link SetKeywordMarkerFilter}
- * if a stem exclusion set is provided, {@link BengaliStemFilter}, and
- * Bengali Stop words
+ * Creates {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} used to tokenize all
+ * the text in the provided {@link Reader}.
+ *
+ * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link
+ * StandardTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter},
+ * {@link IndicNormalizationFilter}, {@link BengaliNormalizationFilter}, {@link
+ * SetKeywordMarkerFilter} if a stem exclusion set is provided, {@link BengaliStemFilter}, and
+ * Bengali Stop words
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new DecimalDigitFilter(result);
- if (!stemExclusionSet.isEmpty())
+ if (!stemExclusionSet.isEmpty()) {
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+ }
result = new IndicNormalizationFilter(result);
result = new BengaliNormalizationFilter(result);
result = new StopFilter(result, stopwords);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java
index 46874b5..effa3c7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilter.java
@@ -16,24 +16,20 @@
*/
package org.apache.lucene.analysis.bn;
-
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import java.io.IOException;
-
/**
- * A {@link TokenFilter} that applies {@link BengaliNormalizer} to normalize the
- * orthography.
- * <p>
- * In some cases the normalization may cause unrelated terms to conflate, so
- * to prevent terms from being normalized use an instance of
- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
- * the {@link KeywordAttribute} before this {@link TokenStream}.
- * </p>
+ * A {@link TokenFilter} that applies {@link BengaliNormalizer} to normalize the orthography.
+ *
+ * <p>In some cases the normalization may cause unrelated terms to conflate, so to prevent terms
+ * from being normalized use an instance of {@link SetKeywordMarkerFilter} or a custom {@link
+ * TokenFilter} that sets the {@link KeywordAttribute} before this {@link TokenStream}.
+ *
* @see BengaliNormalizer
*/
public final class BengaliNormalizationFilter extends TokenFilter {
@@ -41,7 +37,7 @@ public final class BengaliNormalizationFilter extends TokenFilter {
private final BengaliNormalizer normalizer = new BengaliNormalizer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
-
+
public BengaliNormalizationFilter(TokenStream input) {
super(input);
}
@@ -50,10 +46,9 @@ public final class BengaliNormalizationFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAtt.isKeyword())
- termAtt.setLength(normalizer.normalize(termAtt.buffer(),
- termAtt.length()));
+ termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));
return true;
- }
+ }
return false;
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
index 8956e6d..b95cbfb 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.bn;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
- * Factory for {@link BengaliNormalizationFilter}.
+/**
+ * Factory for {@link BengaliNormalizationFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_bnnormal" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -31,6 +30,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.BengaliNormalizationFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 7.1.0
* @lucene.spi {@value #NAME}
*/
@@ -39,13 +39,13 @@ public class BengaliNormalizationFilterFactory extends TokenFilterFactory {
/** SPI name */
public static final String NAME = "bengaliNormalization";
- public BengaliNormalizationFilterFactory(Map<String,String> args) {
+ public BengaliNormalizationFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public BengaliNormalizationFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java
index 1718cbc..c047fbd 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizer.java
@@ -16,23 +16,20 @@
*/
package org.apache.lucene.analysis.bn;
-
import static org.apache.lucene.analysis.util.StemmerUtil.delete;
/**
* Normalizer for Bengali.
- * <p>
- * Implements the Bengali-language specific algorithm specified in:
- * <i>A Double Metaphone encoding for Bangla and its application in spelling checker</i>
- * Naushad UzZaman and Mumit Khan.
+ *
+ * <p>Implements the Bengali-language specific algorithm specified in: <i>A Double Metaphone
+ * encoding for Bangla and its application in spelling checker</i> Naushad UzZaman and Mumit Khan.
* http://www.panl10n.net/english/final%20reports/pdf%20files/Bangladesh/BAN16.pdf
- * </p>
*/
public class BengaliNormalizer {
/**
* Normalize an input buffer of Bengali text
*
- * @param s input buffer
+ * @param s input buffer
* @param len length of input buffer
* @return length of input buffer after normalization
*/
@@ -40,102 +37,103 @@ public class BengaliNormalizer {
for (int i = 0; i < len; i++) {
switch (s[i]) {
- // delete Chandrabindu
+ // delete Chandrabindu
case '\u0981':
len = delete(s, i, len);
i--;
break;
- // DirghoI kar -> RosshoI kar
+ // DirghoI kar -> RosshoI kar
case '\u09C0':
s[i] = '\u09BF';
break;
- // DirghoU kar -> RosshoU kar
+ // DirghoU kar -> RosshoU kar
case '\u09C2':
s[i] = '\u09C1';
break;
- // Khio (Ka + Hoshonto + Murdorno Sh)
+ // Khio (Ka + Hoshonto + Murdorno Sh)
case '\u0995':
- if(i + 2 < len && s[i+1] == '\u09CD' && s[i+2] == '\u09BF') {
+ if (i + 2 < len && s[i + 1] == '\u09CD' && s[i + 2] == '\u09BF') {
if (i == 0) {
s[i] = '\u0996';
len = delete(s, i + 2, len);
len = delete(s, i + 1, len);
} else {
- s[i+1] = '\u0996';
+ s[i + 1] = '\u0996';
len = delete(s, i + 2, len);
}
}
break;
- // Nga to Anusvara
+ // Nga to Anusvara
case '\u0999':
s[i] = '\u0982';
break;
- // Ja Phala
+ // Ja Phala
case '\u09AF':
- if(i - 2 == 0 && s[i-1] == '\u09CD') {
+ if (i - 2 == 0 && s[i - 1] == '\u09CD') {
s[i - 1] = '\u09C7';
- if(i + 1 < len && s[i+1] == '\u09BE') {
- len = delete(s, i+1, len);
+ if (i + 1 < len && s[i + 1] == '\u09BE') {
+ len = delete(s, i + 1, len);
}
len = delete(s, i, len);
- i --;
- } else if(i - 1 >= 0 && s[i-1] == '\u09CD' ){
+ i--;
+ } else if (i - 1 >= 0 && s[i - 1] == '\u09CD') {
len = delete(s, i, len);
- len = delete(s, i-1, len);
- i -=2;
+ len = delete(s, i - 1, len);
+ i -= 2;
}
break;
- // Ba Phalaa
+ // Ba Phalaa
case '\u09AC':
- if((i >= 1 && s[i-1] != '\u09CD') || i == 0)
+ if ((i >= 1 && s[i - 1] != '\u09CD') || i == 0) {
break;
- if(i - 2 == 0) {
+ }
+ if (i - 2 == 0) {
len = delete(s, i, len);
len = delete(s, i - 1, len);
i -= 2;
- } else if(i - 5 >= 0 && s[i - 3] == '\u09CD') {
+ } else if (i - 5 >= 0 && s[i - 3] == '\u09CD') {
len = delete(s, i, len);
- len = delete(s, i-1, len);
- i -=2;
- } else if(i - 2 >= 0){
+ len = delete(s, i - 1, len);
+ i -= 2;
+ } else if (i - 2 >= 0) {
s[i - 1] = s[i - 2];
len = delete(s, i, len);
- i --;
+ i--;
}
break;
- // Visarga
+ // Visarga
case '\u0983':
- if(i == len -1) {
- if(len <= 3) {
+ if (i == len - 1) {
+ if (len <= 3) {
s[i] = '\u09B9';
} else {
len = delete(s, i, len);
}
} else {
- s[i] = s[i+1];
+ s[i] = s[i + 1];
}
break;
- //All sh
+ // All sh
case '\u09B6':
case '\u09B7':
s[i] = '\u09B8';
break;
- //check na
+ // check na
case '\u09A3':
s[i] = '\u09A8';
break;
- //check ra
+ // check ra
case '\u09DC':
case '\u09DD':
s[i] = '\u09B0';
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java
index 9787027..49cfd38 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilter.java
@@ -16,31 +16,28 @@
*/
package org.apache.lucene.analysis.bn;
-
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import java.io.IOException;
-
-/**
- * A {@link TokenFilter} that applies {@link BengaliStemmer} to stem Bengali words.
- */
+/** A {@link TokenFilter} that applies {@link BengaliStemmer} to stem Bengali words. */
public final class BengaliStemFilter extends TokenFilter {
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);
private final BengaliStemmer bengaliStemmer = new BengaliStemmer();
-
+
public BengaliStemFilter(TokenStream input) {
super(input);
}
-
+
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAttribute.isKeyword())
- termAttribute.setLength(bengaliStemmer.stem(termAttribute.buffer(), termAttribute.length()));
+ termAttribute.setLength(
+ bengaliStemmer.stem(termAttribute.buffer(), termAttribute.length()));
return true;
} else {
return false;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java
index ca84aca..a8711b4 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.bn;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.TokenFilterFactory;
-
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
- * Factory for {@link BengaliStemFilter}.
+/**
+ * Factory for {@link BengaliStemFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_histem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -31,6 +30,7 @@ import java.util.Map;
* <filter class="solr.BengaliStemFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 7.1.0
* @lucene.spi {@value #NAME}
*/
@@ -39,13 +39,13 @@ public class BengaliStemFilterFactory extends TokenFilterFactory {
/** SPI name */
public static final String NAME = "bengaliStem";
- public BengaliStemFilterFactory(Map<String,String> args) {
+ public BengaliStemFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public BengaliStemFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java
index 8bc555a..e07521c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliStemmer.java
@@ -16,167 +16,148 @@
*/
package org.apache.lucene.analysis.bn;
-
import static org.apache.lucene.analysis.util.StemmerUtil.endsWith;
/**
* Stemmer for Bengali.
- * <p>
- * The algorithm is based on the report in:
- * <i>Natural Language Processing in an Indian Language (Bengali)-I: Verb Phrase Analysis</i>
- * P Sengupta and B B Chaudhuri
- * </p>
*
- * <p>
- * Few Stemmer criteria are taken from:
- * <i>http://members.unine.ch/jacques.savoy/clef/BengaliStemmerLight.java.txt</i>
- * </p>
+ * <p>The algorithm is based on the report in: <i>Natural Language Processing in an Indian Language
+ * (Bengali)-I: Verb Phrase Analysis</i> P Sengupta and B B Chaudhuri
+ *
+ * <p>Few Stemmer criteria are taken from:
+ * <i>http://members.unine.ch/jacques.savoy/clef/BengaliStemmerLight.java.txt</i>
*/
public class BengaliStemmer {
public int stem(char buffer[], int len) {
// 8
- if (len > 9 && (endsWith(buffer, len, "িয়াছিলাম")
- || endsWith(buffer, len, "িতেছিলাম")
- || endsWith(buffer, len, "িতেছিলেন")
- || endsWith(buffer, len, "ইতেছিলেন")
- || endsWith(buffer, len, "িয়াছিলেন")
- || endsWith(buffer, len, "ইয়াছিলেন")
- ))
- return len - 8;
+ if (len > 9
+ && (endsWith(buffer, len, "িয়াছিলাম")
+ || endsWith(buffer, len, "িতেছিলাম")
+ || endsWith(buffer, len, "িতেছিলেন")
+ || endsWith(buffer, len, "ইতেছিলেন")
+ || endsWith(buffer, len, "িয়াছিলেন")
+ || endsWith(buffer, len, "ইয়াছিলেন"))) return len - 8;
// 7
- if ((len > 8) && (endsWith(buffer, len, "িতেছিলি")
- || endsWith(buffer, len, "িতেছিলে")
- || endsWith(buffer, len, "িয়াছিলা")
- || endsWith(buffer, len, "িয়াছিলে")
- || endsWith(buffer, len, "িতেছিলা")
- || endsWith(buffer, len, "িয়াছিলি")
-
- || endsWith(buffer, len, "য়েদেরকে")
- ))
- return len - 7;
+ if ((len > 8)
+ && (endsWith(buffer, len, "িতেছিলি")
+ || endsWith(buffer, len, "িতেছিলে")
+ || endsWith(buffer, len, "িয়াছিলা")
+ || endsWith(buffer, len, "িয়াছিলে")
+ || endsWith(buffer, len, "িতেছিলা")
+ || endsWith(buffer, len, "িয়াছিলি")
+ || endsWith(buffer, len, "য়েদেরকে"))) return len - 7;
// 6
- if ((len > 7) && (endsWith(buffer, len, "িতেছিস")
- || endsWith(buffer, len, "িতেছেন")
- || endsWith(buffer, len, "িয়াছিস")
- || endsWith(buffer, len, "িয়াছেন")
- || endsWith(buffer, len, "েছিলাম")
- || endsWith(buffer, len, "েছিলেন")
-
- || endsWith(buffer, len, "েদেরকে")
- ))
- return len - 6;
+ if ((len > 7)
+ && (endsWith(buffer, len, "িতেছিস")
+ || endsWith(buffer, len, "িতেছেন")
+ || endsWith(buffer, len, "িয়াছিস")
+ || endsWith(buffer, len, "িয়াছেন")
+ || endsWith(buffer, len, "েছিলাম")
+ || endsWith(buffer, len, "েছিলেন")
+ || endsWith(buffer, len, "েদেরকে"))) return len - 6;
// 5
- if ((len > 6) && (endsWith(buffer, len, "িতেছি")
- || endsWith(buffer, len, "িতেছা")
- || endsWith(buffer, len, "িতেছে")
- || endsWith(buffer, len, "ছিলাম")
- || endsWith(buffer, len, "ছিলেন")
- || endsWith(buffer, len, "িয়াছি")
- || endsWith(buffer, len, "িয়াছা")
- || endsWith(buffer, len, "িয়াছে")
- || endsWith(buffer, len, "েছিলে")
- || endsWith(buffer, len, "েছিলা")
-
- || endsWith(buffer, len, "য়েদের")
- || endsWith(buffer, len, "দেরকে")
- ))
- return len - 5;
+ if ((len > 6)
+ && (endsWith(buffer, len, "িতেছি")
+ || endsWith(buffer, len, "িতেছা")
+ || endsWith(buffer, len, "িতেছে")
+ || endsWith(buffer, len, "ছিলাম")
+ || endsWith(buffer, len, "ছিলেন")
+ || endsWith(buffer, len, "িয়াছি")
+ || endsWith(buffer, len, "িয়াছা")
+ || endsWith(buffer, len, "িয়াছে")
+ || endsWith(buffer, len, "েছিলে")
+ || endsWith(buffer, len, "েছিলা")
+ || endsWith(buffer, len, "য়েদের")
+ || endsWith(buffer, len, "দেরকে"))) return len - 5;
// 4
- if ((len > 5) && (endsWith(buffer, len, "িলাম")
- || endsWith(buffer, len, "িলেন")
- || endsWith(buffer, len, "িতাম")
- || endsWith(buffer, len, "িতেন")
- || endsWith(buffer, len, "িবেন")
- || endsWith(buffer, len, "ছিলি")
- || endsWith(buffer, len, "ছিলে")
- || endsWith(buffer, len, "ছিলা")
- || endsWith(buffer, len, "তেছে")
- || endsWith(buffer, len, "িতেছ")
-
- || endsWith(buffer, len, "খানা")
- || endsWith(buffer, len, "খানি")
- || endsWith(buffer, len, "গুলো")
- || endsWith(buffer, len, "গুলি")
- || endsWith(buffer, len, "য়েরা")
- || endsWith(buffer, len, "েদের")
- ))
- return len - 4;
+ if ((len > 5)
+ && (endsWith(buffer, len, "িলাম")
+ || endsWith(buffer, len, "িলেন")
+ || endsWith(buffer, len, "িতাম")
+ || endsWith(buffer, len, "িতেন")
+ || endsWith(buffer, len, "িবেন")
+ || endsWith(buffer, len, "ছিলি")
+ || endsWith(buffer, len, "ছিলে")
+ || endsWith(buffer, len, "ছিলা")
+ || endsWith(buffer, len, "তেছে")
+ || endsWith(buffer, len, "িতেছ")
+ || endsWith(buffer, len, "খানা")
+ || endsWith(buffer, len, "খানি")
+ || endsWith(buffer, len, "গুলো")
+ || endsWith(buffer, len, "গুলি")
+ || endsWith(buffer, len, "য়েরা")
+ || endsWith(buffer, len, "েদের"))) return len - 4;
// 3
- if ((len > 4) && (endsWith(buffer, len, "লাম")
- || endsWith(buffer, len, "িলি")
- || endsWith(buffer, len, "ইলি")
- || endsWith(buffer, len, "িলে")
- || endsWith(buffer, len, "ইলে")
- || endsWith(buffer, len, "লেন")
- || endsWith(buffer, len, "িলা")
- || endsWith(buffer, len, "ইলা")
- || endsWith(buffer, len, "তাম")
- || endsWith(buffer, len, "িতি")
- || endsWith(buffer, len, "ইতি")
- || endsWith(buffer, len, "িতে")
- || endsWith(buffer, len, "ইতে")
- || endsWith(buffer, len, "তেন")
- || endsWith(buffer, len, "িতা")
- || endsWith(buffer, len, "িবা")
- || endsWith(buffer, len, "ইবা")
- || endsWith(buffer, len, "িবি")
- || endsWith(buffer, len, "ইবি")
- || endsWith(buffer, len, "বেন")
- || endsWith(buffer, len, "িবে")
- || endsWith(buffer, len, "ইবে")
- || endsWith(buffer, len, "ছেন")
-
- || endsWith(buffer, len, "য়োন")
- || endsWith(buffer, len, "য়ের")
- || endsWith(buffer, len, "েরা")
- || endsWith(buffer, len, "দের")
- ))
- return len - 3;
+ if ((len > 4)
+ && (endsWith(buffer, len, "লাম")
+ || endsWith(buffer, len, "িলি")
+ || endsWith(buffer, len, "ইলি")
+ || endsWith(buffer, len, "িলে")
+ || endsWith(buffer, len, "ইলে")
+ || endsWith(buffer, len, "লেন")
+ || endsWith(buffer, len, "িলা")
+ || endsWith(buffer, len, "ইলা")
+ || endsWith(buffer, len, "তাম")
+ || endsWith(buffer, len, "িতি")
+ || endsWith(buffer, len, "ইতি")
+ || endsWith(buffer, len, "িতে")
+ || endsWith(buffer, len, "ইতে")
+ || endsWith(buffer, len, "তেন")
+ || endsWith(buffer, len, "িতা")
+ || endsWith(buffer, len, "িবা")
+ || endsWith(buffer, len, "ইবা")
+ || endsWith(buffer, len, "িবি")
+ || endsWith(buffer, len, "ইবি")
+ || endsWith(buffer, len, "বেন")
+ || endsWith(buffer, len, "িবে")
+ || endsWith(buffer, len, "ইবে")
+ || endsWith(buffer, len, "ছেন")
+ || endsWith(buffer, len, "য়োন")
+ || endsWith(buffer, len, "য়ের")
+ || endsWith(buffer, len, "েরা")
+ || endsWith(buffer, len, "দের"))) return len - 3;
// 2
- if ((len > 3) && (endsWith(buffer, len, "িস")
- || endsWith(buffer, len, "েন")
- || endsWith(buffer, len, "লি")
- || endsWith(buffer, len, "লে")
- || endsWith(buffer, len, "লা")
- || endsWith(buffer, len, "তি")
- || endsWith(buffer, len, "তে")
- || endsWith(buffer, len, "তা")
- || endsWith(buffer, len, "বি")
- || endsWith(buffer, len, "বে")
- || endsWith(buffer, len, "বা")
- || endsWith(buffer, len, "ছি")
- || endsWith(buffer, len, "ছা")
- || endsWith(buffer, len, "ছে")
- || endsWith(buffer, len, "ুন")
- || endsWith(buffer, len, "ুক")
-
- || endsWith(buffer, len, "টা")
- || endsWith(buffer, len, "টি")
- || endsWith(buffer, len, "নি")
- || endsWith(buffer, len, "ের")
- || endsWith(buffer, len, "তে")
- || endsWith(buffer, len, "রা")
- || endsWith(buffer, len, "কে")
- ))
- return len - 2;
+ if ((len > 3)
+ && (endsWith(buffer, len, "িস")
+ || endsWith(buffer, len, "েন")
+ || endsWith(buffer, len, "লি")
+ || endsWith(buffer, len, "লে")
+ || endsWith(buffer, len, "লা")
+ || endsWith(buffer, len, "তি")
+ || endsWith(buffer, len, "তে")
+ || endsWith(buffer, len, "তা")
+ || endsWith(buffer, len, "বি")
+ || endsWith(buffer, len, "বে")
+ || endsWith(buffer, len, "বা")
+ || endsWith(buffer, len, "ছি")
+ || endsWith(buffer, len, "ছা")
+ || endsWith(buffer, len, "ছে")
+ || endsWith(buffer, len, "ুন")
+ || endsWith(buffer, len, "ুক")
+ || endsWith(buffer, len, "টা")
+ || endsWith(buffer, len, "টি")
+ || endsWith(buffer, len, "নি")
+ || endsWith(buffer, len, "ের")
+ || endsWith(buffer, len, "তে")
+ || endsWith(buffer, len, "রা")
+ || endsWith(buffer, len, "কে"))) return len - 2;
// 1
- if ((len > 2) && (endsWith(buffer, len, "ি")
- || endsWith(buffer, len, "ী")
- || endsWith(buffer, len, "া")
- || endsWith(buffer, len, "ো")
- || endsWith(buffer, len, "ে")
- || endsWith(buffer, len, "ব")
- || endsWith(buffer, len, "ত")
- ))
- return len - 1;
+ if ((len > 2)
+ && (endsWith(buffer, len, "ি")
+ || endsWith(buffer, len, "ী")
+ || endsWith(buffer, len, "া")
+ || endsWith(buffer, len, "ো")
+ || endsWith(buffer, len, "ে")
+ || endsWith(buffer, len, "ব")
+ || endsWith(buffer, len, "ত"))) return len - 1;
return len;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java
index eea39a9..d272858 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analyzer for Bengali Language.
- */
+/** Analyzer for Bengali Language. */
package org.apache.lucene.analysis.bn;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilter.java
index c37f7d7..b70768e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilter.java
@@ -16,21 +16,19 @@
*/
package org.apache.lucene.analysis.boost;
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BoostAttribute;
-import java.io.IOException;
-
-
/**
* Characters before the delimiter are the "token", those after are the boost.
- * <p>
- * For example, if the delimiter is '|', then for the string "foo|0.7", foo is the token
- * and 0.7 is the boost.
- * <p>
- * Note make sure your Tokenizer doesn't split on the delimiter, or this won't work
+ *
+ * <p>For example, if the delimiter is '|', then for the string "foo|0.7", foo is the token and 0.7
+ * is the boost.
+ *
+ * <p>Note make sure your Tokenizer doesn't split on the delimiter, or this won't work
*/
public final class DelimitedBoostTokenFilter extends TokenFilter {
private final char delimiter;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilterFactory.java
index 7965ece..71f42bd 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/DelimitedBoostTokenFilterFactory.java
@@ -16,13 +16,13 @@
*/
package org.apache.lucene.analysis.boost;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.TokenFilterFactory;
-
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link DelimitedBoostTokenFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_dlmtd" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -35,18 +35,15 @@ import java.util.Map;
*/
public class DelimitedBoostTokenFilterFactory extends TokenFilterFactory {
- /**
- * SPI name
- */
+ /** SPI name */
public static final String NAME = "delimitedBoost";
+
public static final String DELIMITER_ATTR = "delimiter";
public static final char DEFAULT_DELIMITER = '|';
private final char delimiter;
- /**
- * Creates a new DelimitedPayloadTokenFilterFactory
- */
+ /** Creates a new DelimitedPayloadTokenFilterFactory */
public DelimitedBoostTokenFilterFactory(Map<String, String> args) {
super(args);
delimiter = getChar(args, DELIMITER_ATTR, DEFAULT_DELIMITER);
@@ -64,5 +61,4 @@ public class DelimitedBoostTokenFilterFactory extends TokenFilterFactory {
public DelimitedBoostTokenFilter create(TokenStream input) {
return new DelimitedBoostTokenFilter(input, delimiter);
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/package-info.java
index 9bae5dc..074baba 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/boost/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Provides various convenience classes for creating boosts on Tokens.
- */
+/** Provides various convenience classes for creating boosts on Tokens. */
package org.apache.lucene.analysis.boost;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
index 93c565f..0520ba7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
@@ -16,11 +16,9 @@
*/
package org.apache.lucene.analysis.br;
-
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -35,37 +33,39 @@ import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
- * {@link Analyzer} for Brazilian Portuguese language.
- * <p>
- * Supports an external list of stopwords (words that
- * will not be indexed at all) and an external list of exclusions (words that will
- * not be stemmed, but indexed).
- * </p>
+ * {@link Analyzer} for Brazilian Portuguese language.
+ *
+ * <p>Supports an external list of stopwords (words that will not be indexed at all) and an external
+ * list of exclusions (words that will not be stemmed, but indexed).
*
- * <p><b>NOTE</b>: This class uses the same {@link org.apache.lucene.util.Version}
- * dependent settings as {@link StandardAnalyzer}.</p>
+ * <p><b>NOTE</b>: This class uses the same {@link org.apache.lucene.util.Version} dependent
+ * settings as {@link StandardAnalyzer}.
*
* @since 3.1
*/
public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
/** File containing default Brazilian Portuguese stopwords. */
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
-
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
/**
* Returns an unmodifiable instance of the default stop-words set.
+ *
* @return an unmodifiable instance of the default stop-words set.
*/
- public static CharArraySet getDefaultStopSet(){
+ public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
-
+
static {
try {
- DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(BrazilianAnalyzer.class,
- DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8), "#");
+ DEFAULT_STOP_SET =
+ WordlistLoader.getWordSet(
+ IOUtils.getDecodingReader(
+ BrazilianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
+ "#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@@ -74,34 +74,27 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
}
}
-
- /**
- * Contains words that should be indexed but not stemmed.
- */
+ /** Contains words that should be indexed but not stemmed. */
private CharArraySet excltable = CharArraySet.EMPTY_SET;
- /**
- * Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}).
- */
+ /** Builds an analyzer with the default stop words ({@link #getDefaultStopSet()}). */
public BrazilianAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
/**
* Builds an analyzer with the given stop words
- *
- * @param stopwords
- * a stopword set
+ *
+ * @param stopwords a stopword set
*/
public BrazilianAnalyzer(CharArraySet stopwords) {
- super(stopwords);
+ super(stopwords);
}
/**
* Builds an analyzer with the given stop words and stemming exclusion words
- *
- * @param stopwords
- * a stopword set
+ *
+ * @param stopwords a stopword set
*/
public BrazilianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
this(stopwords);
@@ -109,21 +102,19 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
}
/**
- * Creates
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * used to tokenize all the text in the provided {@link Reader}.
- *
- * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from a {@link StandardTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link StopFilter}
- * , and {@link BrazilianStemFilter}.
+ * Creates {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} used to tokenize all
+ * the text in the provided {@link Reader}.
+ *
+ * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link
+ * StandardTokenizer} filtered with {@link LowerCaseFilter}, {@link StopFilter}, and {@link
+ * BrazilianStemFilter}.
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer source = new StandardTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new StopFilter(result, stopwords);
- if(excltable != null && !excltable.isEmpty())
+ if (excltable != null && !excltable.isEmpty())
result = new SetKeywordMarkerFilter(result, excltable);
return new TokenStreamComponents(source, new BrazilianStemFilter(result));
}
@@ -133,4 +124,3 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
return new LowerCaseFilter(in);
}
}
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
index e605df5..6e8ff1f 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.br;
-
import java.io.IOException;
import java.util.Set;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
@@ -28,33 +26,31 @@ import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
/**
* A {@link TokenFilter} that applies {@link BrazilianStemmer}.
- * <p>
- * To prevent terms from being stemmed use an instance of
- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
- * the {@link KeywordAttribute} before this {@link TokenStream}.
- * </p>
+ *
+ * <p>To prevent terms from being stemmed use an instance of {@link SetKeywordMarkerFilter} or a
+ * custom {@link TokenFilter} that sets the {@link KeywordAttribute} before this {@link
+ * TokenStream}.
+ *
* @see SetKeywordMarkerFilter
- *
*/
public final class BrazilianStemFilter extends TokenFilter {
- /**
- * {@link BrazilianStemmer} in use by this filter.
- */
+ /** {@link BrazilianStemmer} in use by this filter. */
private BrazilianStemmer stemmer = new BrazilianStemmer();
+
private Set<?> exclusions = null;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
/**
- * Creates a new BrazilianStemFilter
- *
- * @param in the source {@link TokenStream}
+ * Creates a new BrazilianStemFilter
+ *
+ * @param in the source {@link TokenStream}
*/
public BrazilianStemFilter(TokenStream in) {
super(in);
}
-
+
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
@@ -63,8 +59,7 @@ public final class BrazilianStemFilter extends TokenFilter {
if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) {
final String s = stemmer.stem(term);
// If not stemmed, don't waste the time adjusting the token.
- if ((s != null) && !s.equals(term))
- termAtt.setEmpty().append(s);
+ if ((s != null) && !s.equals(term)) termAtt.setEmpty().append(s);
}
return true;
} else {
@@ -72,5 +67,3 @@ public final class BrazilianStemFilter extends TokenFilter {
}
}
}
-
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilterFactory.java
index b88446f..634e1cc 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.br;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
+/**
* Factory for {@link BrazilianStemFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_brstem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -40,15 +39,15 @@ public class BrazilianStemFilterFactory extends TokenFilterFactory {
/** SPI name */
public static final String NAME = "brazilianStem";
-
+
/** Creates a new BrazilianStemFilterFactory */
- public BrazilianStemFilterFactory(Map<String,String> args) {
+ public BrazilianStemFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public BrazilianStemFilterFactory() {
throw defaultCtorException();
@@ -59,4 +58,3 @@ public class BrazilianStemFilterFactory extends TokenFilterFactory {
return new BrazilianStemFilter(in);
}
}
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java
index 2737358..6958cbe 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java
@@ -18,53 +18,47 @@ package org.apache.lucene.analysis.br;
import java.util.Locale;
-
-/**
- * A stemmer for Brazilian Portuguese words.
- */
+/** A stemmer for Brazilian Portuguese words. */
public class BrazilianStemmer {
private static final Locale locale = new Locale("pt", "BR");
- /**
- * Changed term
- */
- private String TERM ;
- private String CT ;
- private String R1 ;
- private String R2 ;
- private String RV ;
+ /** Changed term */
+ private String TERM;
+ private String CT;
+ private String R1;
+ private String R2;
+ private String RV;
- public BrazilianStemmer() {
- }
+ public BrazilianStemmer() {}
/**
* Stems the given term to an unique <code>discriminator</code>.
*
- * @param term The term that should be stemmed.
- * @return Discriminator for <code>term</code>
+ * @param term The term that should be stemmed.
+ * @return Discriminator for <code>term</code>
*/
- protected String stem( String term ) {
- boolean altered = false ; // altered the term
+ protected String stem(String term) {
+ boolean altered = false; // altered the term
// creates CT
- createCT(term) ;
+ createCT(term);
- if ( !isIndexable( CT ) ) {
+ if (!isIndexable(CT)) {
return null;
}
- if ( !isStemmable( CT ) ) {
- return CT ;
+ if (!isStemmable(CT)) {
+ return CT;
}
- R1 = getR1(CT) ;
- R2 = getR1(R1) ;
- RV = getRV(CT) ;
- TERM = term + ";" +CT ;
+ R1 = getR1(CT);
+ R2 = getR1(R1);
+ RV = getRV(CT);
+ TERM = term + ";" + CT;
- altered = step1() ;
+ altered = step1();
if (!altered) {
- altered = step2() ;
+ altered = step2();
}
if (altered) {
@@ -73,20 +67,20 @@ public class BrazilianStemmer {
step4();
}
- step5() ;
+ step5();
- return CT ;
+ return CT;
}
/**
* Checks a term if it can be processed correctly.
*
- * @return true if, and only if, the given term consists in letters.
+ * @return true if, and only if, the given term consists in letters.
*/
- private boolean isStemmable( String term ) {
- for ( int c = 0; c < term.length(); c++ ) {
+ private boolean isStemmable(String term) {
+ for (int c = 0; c < term.length(); c++) {
// Discard terms that contain non-letter characters.
- if ( !Character.isLetter(term.charAt(c))) {
+ if (!Character.isLetter(term.charAt(c))) {
return false;
}
}
@@ -96,10 +90,10 @@ public class BrazilianStemmer {
/**
* Checks a term if it can be processed indexed.
*
- * @return true if it can be indexed
+ * @return true if it can be indexed
*/
- private boolean isIndexable( String term ) {
- return (term.length() < 30) && (term.length() > 2) ;
+ private boolean isIndexable(String term) {
+ return (term.length() < 30) && (term.length() > 2);
}
/**
@@ -107,179 +101,165 @@ public class BrazilianStemmer {
*
* @return true if is vowel
*/
- private boolean isVowel( char value ) {
- return (value == 'a') ||
- (value == 'e') ||
- (value == 'i') ||
- (value == 'o') ||
- (value == 'u') ;
+ private boolean isVowel(char value) {
+ return (value == 'a') || (value == 'e') || (value == 'i') || (value == 'o') || (value == 'u');
}
/**
* Gets R1
*
- * R1 - is the region after the first non-vowel following a vowel,
- * or is the null region at the end of the word if there is
- * no such non-vowel.
+ * <p>R1 - is the region after the first non-vowel following a vowel, or is the null region at the
+ * end of the word if there is no such non-vowel.
*
* @return null or a string representing R1
*/
- private String getR1( String value ) {
- int i;
- int j;
+ private String getR1(String value) {
+ int i;
+ int j;
// be-safe !!!
if (value == null) {
- return null ;
+ return null;
}
// find 1st vowel
- i = value.length()-1 ;
- for (j=0 ; j < i ; j++) {
+ i = value.length() - 1;
+ for (j = 0; j < i; j++) {
if (isVowel(value.charAt(j))) {
- break ;
+ break;
}
}
if (!(j < i)) {
- return null ;
+ return null;
}
// find 1st non-vowel
- for ( ; j < i ; j++) {
+ for (; j < i; j++) {
if (!(isVowel(value.charAt(j)))) {
- break ;
+ break;
}
}
if (!(j < i)) {
- return null ;
+ return null;
}
- return value.substring(j+1) ;
+ return value.substring(j + 1);
}
/**
* Gets RV
*
- * RV - IF the second letter is a consonant, RV is the region after
- * the next following vowel,
+ * <p>RV - IF the second letter is a consonant, RV is the region after the next following vowel,
*
- * OR if the first two letters are vowels, RV is the region
- * after the next consonant,
+ * <p>OR if the first two letters are vowels, RV is the region after the next consonant,
*
- * AND otherwise (consonant-vowel case) RV is the region after
- * the third letter.
+ * <p>AND otherwise (consonant-vowel case) RV is the region after the third letter.
*
- * BUT RV is the end of the word if this positions cannot be
- * found.
+ * <p>BUT RV is the end of the word if this positions cannot be found.
*
* @return null or a string representing RV
*/
- private String getRV( String value ) {
- int i;
- int j;
+ private String getRV(String value) {
+ int i;
+ int j;
// be-safe !!!
if (value == null) {
- return null ;
+ return null;
}
- i = value.length()-1 ;
+ i = value.length() - 1;
// RV - IF the second letter is a consonant, RV is the region after
// the next following vowel,
if ((i > 0) && !isVowel(value.charAt(1))) {
// find 1st vowel
- for (j=2 ; j < i ; j++) {
+ for (j = 2; j < i; j++) {
if (isVowel(value.charAt(j))) {
- break ;
+ break;
}
}
if (j < i) {
- return value.substring(j+1) ;
+ return value.substring(j + 1);
}
}
-
// RV - OR if the first two letters are vowels, RV is the region
// after the next consonant,
- if ((i > 1) &&
- isVowel(value.charAt(0)) &&
- isVowel(value.charAt(1))) {
+ if ((i > 1) && isVowel(value.charAt(0)) && isVowel(value.charAt(1))) {
// find 1st consoant
- for (j=2 ; j < i ; j++) {
+ for (j = 2; j < i; j++) {
if (!isVowel(value.charAt(j))) {
- break ;
+ break;
}
}
if (j < i) {
- return value.substring(j+1) ;
+ return value.substring(j + 1);
}
}
// RV - AND otherwise (consonant-vowel case) RV is the region after
// the third letter.
if (i > 2) {
- return value.substring(3) ;
+ return value.substring(3);
}
- return null ;
+ return null;
}
/**
- * 1) Turn to lowercase
- * 2) Remove accents
- * 3) ã -> a ; õ -> o
- * 4) ç -> c
+ * 1) Turn to lowercase 2) Remove accents 3) ã -> a ; õ -> o 4) ç -> c
*
* @return null or a string transformed
*/
- private String changeTerm( String value ) {
- int j;
- String r = "" ;
+ private String changeTerm(String value) {
+ int j;
+ String r = "";
// be-safe !!!
if (value == null) {
- return null ;
+ return null;
}
- value = value.toLowerCase(locale) ;
- for (j=0 ; j < value.length() ; j++) {
- if ((value.charAt(j) == 'á') ||
- (value.charAt(j) == 'â') ||
- (value.charAt(j) == 'ã')) {
- r= r + "a" ; continue ;
+ value = value.toLowerCase(locale);
+ for (j = 0; j < value.length(); j++) {
+ if ((value.charAt(j) == 'á') || (value.charAt(j) == 'â') || (value.charAt(j) == 'ã')) {
+ r = r + "a";
+ continue;
}
- if ((value.charAt(j) == 'é') ||
- (value.charAt(j) == 'ê')) {
- r= r + "e" ; continue ;
+ if ((value.charAt(j) == 'é') || (value.charAt(j) == 'ê')) {
+ r = r + "e";
+ continue;
}
if (value.charAt(j) == 'í') {
- r= r + "i" ; continue ;
+ r = r + "i";
+ continue;
}
- if ((value.charAt(j) == 'ó') ||
- (value.charAt(j) == 'ô') ||
- (value.charAt(j) == 'õ')) {
- r= r + "o" ; continue ;
+ if ((value.charAt(j) == 'ó') || (value.charAt(j) == 'ô') || (value.charAt(j) == 'õ')) {
+ r = r + "o";
+ continue;
}
- if ((value.charAt(j) == 'ú') ||
- (value.charAt(j) == 'ü')) {
- r= r + "u" ; continue ;
+ if ((value.charAt(j) == 'ú') || (value.charAt(j) == 'ü')) {
+ r = r + "u";
+ continue;
}
if (value.charAt(j) == 'ç') {
- r= r + "c" ; continue ;
+ r = r + "c";
+ continue;
}
if (value.charAt(j) == 'ñ') {
- r= r + "n" ; continue ;
+ r = r + "n";
+ continue;
}
- r= r+ value.charAt(j) ;
+ r = r + value.charAt(j);
}
- return r ;
+ return r;
}
/**
@@ -287,18 +267,18 @@ public class BrazilianStemmer {
*
* @return true if the string ends with the specified suffix
*/
- private boolean suffix( String value, String suffix ) {
+ private boolean suffix(String value, String suffix) {
// be-safe !!!
if ((value == null) || (suffix == null)) {
- return false ;
+ return false;
}
if (suffix.length() > value.length()) {
- return false ;
+ return false;
}
- return value.substring(value.length()-suffix.length()).equals(suffix);
+ return value.substring(value.length() - suffix.length()).equals(suffix);
}
/**
@@ -306,22 +286,20 @@ public class BrazilianStemmer {
*
* @return the replaced String
*/
- private String replaceSuffix( String value, String toReplace, String changeTo ) {
- String vvalue ;
+ private String replaceSuffix(String value, String toReplace, String changeTo) {
+ String vvalue;
// be-safe !!!
- if ((value == null) ||
- (toReplace == null) ||
- (changeTo == null) ) {
- return value ;
+ if ((value == null) || (toReplace == null) || (changeTo == null)) {
+ return value;
}
- vvalue = removeSuffix(value,toReplace) ;
+ vvalue = removeSuffix(value, toReplace);
if (value.equals(vvalue)) {
- return value ;
+ return value;
} else {
- return vvalue + changeTo ;
+ return vvalue + changeTo;
}
}
@@ -330,15 +308,13 @@ public class BrazilianStemmer {
*
* @return the String without the suffix
*/
- private String removeSuffix( String value, String toRemove ) {
+ private String removeSuffix(String value, String toRemove) {
// be-safe !!!
- if ((value == null) ||
- (toRemove == null) ||
- !suffix(value,toRemove) ) {
- return value ;
+ if ((value == null) || (toRemove == null) || !suffix(value, toRemove)) {
+ return value;
}
- return value.substring(0,value.length()-toRemove.length()) ;
+ return value.substring(0, value.length() - toRemove.length());
}
/**
@@ -346,679 +322,823 @@ public class BrazilianStemmer {
*
* @return true if the suffix is preceded
*/
- private boolean suffixPreceded( String value, String suffix, String preceded ) {
+ private boolean suffixPreceded(String value, String suffix, String preceded) {
// be-safe !!!
- if ((value == null) ||
- (suffix == null) ||
- (preceded == null) ||
- !suffix(value,suffix) ) {
- return false ;
+ if ((value == null) || (suffix == null) || (preceded == null) || !suffix(value, suffix)) {
+ return false;
}
- return suffix(removeSuffix(value,suffix),preceded) ;
+ return suffix(removeSuffix(value, suffix), preceded);
}
- /**
- * Creates CT (changed term) , substituting * 'ã' and 'õ' for 'a~' and 'o~'.
- */
- private void createCT( String term ) {
- CT = changeTerm(term) ;
+ /** Creates CT (changed term) , substituting * 'ã' and 'õ' for 'a~' and 'o~'. */
+ private void createCT(String term) {
+ CT = changeTerm(term);
- if (CT.length() < 2) return ;
+ if (CT.length() < 2) return;
// if the first character is ... , remove it
- if ((CT.charAt(0) == '"') ||
- (CT.charAt(0) == '\'') ||
- (CT.charAt(0) == '-') ||
- (CT.charAt(0) == ',') ||
- (CT.charAt(0) == ';') ||
- (CT.charAt(0) == '.') ||
- (CT.charAt(0) == '?') ||
- (CT.charAt(0) == '!')
- ) {
- CT = CT.substring(1);
+ if ((CT.charAt(0) == '"')
+ || (CT.charAt(0) == '\'')
+ || (CT.charAt(0) == '-')
+ || (CT.charAt(0) == ',')
+ || (CT.charAt(0) == ';')
+ || (CT.charAt(0) == '.')
+ || (CT.charAt(0) == '?')
+ || (CT.charAt(0) == '!')) {
+ CT = CT.substring(1);
}
- if (CT.length() < 2) return ;
+ if (CT.length() < 2) return;
// if the last character is ... , remove it
- if ((CT.charAt(CT.length()-1) == '-') ||
- (CT.charAt(CT.length()-1) == ',') ||
- (CT.charAt(CT.length()-1) == ';') ||
- (CT.charAt(CT.length()-1) == '.') ||
- (CT.charAt(CT.length()-1) == '?') ||
- (CT.charAt(CT.length()-1) == '!') ||
- (CT.charAt(CT.length()-1) == '\'') ||
- (CT.charAt(CT.length()-1) == '"')
- ) {
- CT = CT.substring(0,CT.length()-1);
+ if ((CT.charAt(CT.length() - 1) == '-')
+ || (CT.charAt(CT.length() - 1) == ',')
+ || (CT.charAt(CT.length() - 1) == ';')
+ || (CT.charAt(CT.length() - 1) == '.')
+ || (CT.charAt(CT.length() - 1) == '?')
+ || (CT.charAt(CT.length() - 1) == '!')
+ || (CT.charAt(CT.length() - 1) == '\'')
+ || (CT.charAt(CT.length() - 1) == '"')) {
+ CT = CT.substring(0, CT.length() - 1);
}
}
-
/**
- * Standard suffix removal.
- * Search for the longest among the following suffixes, and perform
- * the following actions:
+ * Standard suffix removal. Search for the longest among the following suffixes, and perform the
+ * following actions:
*
* @return false if no ending was removed
*/
private boolean step1() {
- if (CT == null) return false ;
+ if (CT == null) return false;
// suffix length = 7
- if (suffix(CT,"uciones") && suffix(R2,"uciones")) {
- CT = replaceSuffix(CT,"uciones","u") ; return true;
+ if (suffix(CT, "uciones") && suffix(R2, "uciones")) {
+ CT = replaceSuffix(CT, "uciones", "u");
+ return true;
}
// suffix length = 6
if (CT.length() >= 6) {
- if (suffix(CT,"imentos") && suffix(R2,"imentos")) {
- CT = removeSuffix(CT,"imentos") ; return true;
+ if (suffix(CT, "imentos") && suffix(R2, "imentos")) {
+ CT = removeSuffix(CT, "imentos");
+ return true;
}
- if (suffix(CT,"amentos") && suffix(R2,"amentos")) {
- CT = removeSuffix(CT,"amentos") ; return true;
+ if (suffix(CT, "amentos") && suffix(R2, "amentos")) {
+ CT = removeSuffix(CT, "amentos");
+ return true;
}
- if (suffix(CT,"adores") && suffix(R2,"adores")) {
- CT = removeSuffix(CT,"adores") ; return true;
+ if (suffix(CT, "adores") && suffix(R2, "adores")) {
+ CT = removeSuffix(CT, "adores");
+ return true;
}
- if (suffix(CT,"adoras") && suffix(R2,"adoras")) {
- CT = removeSuffix(CT,"adoras") ; return true;
+ if (suffix(CT, "adoras") && suffix(R2, "adoras")) {
+ CT = removeSuffix(CT, "adoras");
+ return true;
}
- if (suffix(CT,"logias") && suffix(R2,"logias")) {
- replaceSuffix(CT,"logias","log") ; return true;
+ if (suffix(CT, "logias") && suffix(R2, "logias")) {
+ replaceSuffix(CT, "logias", "log");
+ return true;
}
- if (suffix(CT,"encias") && suffix(R2,"encias")) {
- CT = replaceSuffix(CT,"encias","ente") ; return true;
+ if (suffix(CT, "encias") && suffix(R2, "encias")) {
+ CT = replaceSuffix(CT, "encias", "ente");
+ return true;
}
- if (suffix(CT,"amente") && suffix(R1,"amente")) {
- CT = removeSuffix(CT,"amente") ; return true;
+ if (suffix(CT, "amente") && suffix(R1, "amente")) {
+ CT = removeSuffix(CT, "amente");
+ return true;
}
- if (suffix(CT,"idades") && suffix(R2,"idades")) {
- CT = removeSuffix(CT,"idades") ; return true;
+ if (suffix(CT, "idades") && suffix(R2, "idades")) {
+ CT = removeSuffix(CT, "idades");
+ return true;
}
}
// suffix length = 5
if (CT.length() >= 5) {
- if (suffix(CT,"acoes") && suffix(R2,"acoes")) {
- CT = removeSuffix(CT,"acoes") ; return true;
+ if (suffix(CT, "acoes") && suffix(R2, "acoes")) {
+ CT = removeSuffix(CT, "acoes");
+ return true;
}
- if (suffix(CT,"imento") && suffix(R2,"imento")) {
- CT = removeSuffix(CT,"imento") ; return true;
+ if (suffix(CT, "imento") && suffix(R2, "imento")) {
+ CT = removeSuffix(CT, "imento");
+ return true;
}
- if (suffix(CT,"amento") && suffix(R2,"amento")) {
- CT = removeSuffix(CT,"amento") ; return true;
+ if (suffix(CT, "amento") && suffix(R2, "amento")) {
+ CT = removeSuffix(CT, "amento");
+ return true;
}
- if (suffix(CT,"adora") && suffix(R2,"adora")) {
- CT = removeSuffix(CT,"adora") ; return true;
+ if (suffix(CT, "adora") && suffix(R2, "adora")) {
+ CT = removeSuffix(CT, "adora");
+ return true;
}
- if (suffix(CT,"ismos") && suffix(R2,"ismos")) {
- CT = removeSuffix(CT,"ismos") ; return true;
+ if (suffix(CT, "ismos") && suffix(R2, "ismos")) {
+ CT = removeSuffix(CT, "ismos");
+ return true;
}
- if (suffix(CT,"istas") && suffix(R2,"istas")) {
- CT = removeSuffix(CT,"istas") ; return true;
+ if (suffix(CT, "istas") && suffix(R2, "istas")) {
+ CT = removeSuffix(CT, "istas");
+ return true;
}
- if (suffix(CT,"logia") && suffix(R2,"logia")) {
- CT = replaceSuffix(CT,"logia","log") ; return true;
+ if (suffix(CT, "logia") && suffix(R2, "logia")) {
+ CT = replaceSuffix(CT, "logia", "log");
+ return true;
}
- if (suffix(CT,"ucion") && suffix(R2,"ucion")) {
- CT = replaceSuffix(CT,"ucion","u") ; return true;
+ if (suffix(CT, "ucion") && suffix(R2, "ucion")) {
+ CT = replaceSuffix(CT, "ucion", "u");
+ return true;
}
- if (suffix(CT,"encia") && suffix(R2,"encia")) {
- CT = replaceSuffix(CT,"encia","ente") ; return true;
+ if (suffix(CT, "encia") && suffix(R2, "encia")) {
+ CT = replaceSuffix(CT, "encia", "ente");
+ return true;
}
- if (suffix(CT,"mente") && suffix(R2,"mente")) {
- CT = removeSuffix(CT,"mente") ; return true;
+ if (suffix(CT, "mente") && suffix(R2, "mente")) {
+ CT = removeSuffix(CT, "mente");
+ return true;
}
- if (suffix(CT,"idade") && suffix(R2,"idade")) {
- CT = removeSuffix(CT,"idade") ; return true;
+ if (suffix(CT, "idade") && suffix(R2, "idade")) {
+ CT = removeSuffix(CT, "idade");
+ return true;
}
}
// suffix length = 4
if (CT.length() >= 4) {
- if (suffix(CT,"acao") && suffix(R2,"acao")) {
- CT = removeSuffix(CT,"acao") ; return true;
+ if (suffix(CT, "acao") && suffix(R2, "acao")) {
+ CT = removeSuffix(CT, "acao");
+ return true;
}
- if (suffix(CT,"ezas") && suffix(R2,"ezas")) {
- CT = removeSuffix(CT,"ezas") ; return true;
+ if (suffix(CT, "ezas") && suffix(R2, "ezas")) {
+ CT = removeSuffix(CT, "ezas");
+ return true;
}
- if (suffix(CT,"icos") && suffix(R2,"icos")) {
- CT = removeSuffix(CT,"icos") ; return true ;
+ if (suffix(CT, "icos") && suffix(R2, "icos")) {
+ CT = removeSuffix(CT, "icos");
+ return true;
}
- if (suffix(CT,"icas") && suffix(R2,"icas")) {
- CT = removeSuffix(CT,"icas") ; return true ;
+ if (suffix(CT, "icas") && suffix(R2, "icas")) {
+ CT = removeSuffix(CT, "icas");
+ return true;
}
- if (suffix(CT,"ismo") && suffix(R2,"ismo")) {
- CT = removeSuffix(CT,"ismo") ; return true ;
+ if (suffix(CT, "ismo") && suffix(R2, "ismo")) {
+ CT = removeSuffix(CT, "ismo");
+ return true;
}
- if (suffix(CT,"avel") && suffix(R2,"avel")) {
- CT = removeSuffix(CT,"avel") ; return true ;
+ if (suffix(CT, "avel") && suffix(R2, "avel")) {
+ CT = removeSuffix(CT, "avel");
+ return true;
}
- if (suffix(CT,"ivel") && suffix(R2,"ivel")) {
- CT = removeSuffix(CT,"ivel") ; return true ;
+ if (suffix(CT, "ivel") && suffix(R2, "ivel")) {
+ CT = removeSuffix(CT, "ivel");
+ return true;
}
- if (suffix(CT,"ista") && suffix(R2,"ista")) {
- CT = removeSuffix(CT,"ista") ; return true ;
+ if (suffix(CT, "ista") && suffix(R2, "ista")) {
+ CT = removeSuffix(CT, "ista");
+ return true;
}
- if (suffix(CT,"osos") && suffix(R2,"osos")) {
- CT = removeSuffix(CT,"osos") ; return true ;
+ if (suffix(CT, "osos") && suffix(R2, "osos")) {
+ CT = removeSuffix(CT, "osos");
+ return true;
}
- if (suffix(CT,"osas") && suffix(R2,"osas")) {
- CT = removeSuffix(CT,"osas") ; return true ;
+ if (suffix(CT, "osas") && suffix(R2, "osas")) {
+ CT = removeSuffix(CT, "osas");
+ return true;
}
- if (suffix(CT,"ador") && suffix(R2,"ador")) {
- CT = removeSuffix(CT,"ador") ; return true ;
+ if (suffix(CT, "ador") && suffix(R2, "ador")) {
+ CT = removeSuffix(CT, "ador");
+ return true;
}
- if (suffix(CT,"ivas") && suffix(R2,"ivas")) {
- CT = removeSuffix(CT,"ivas") ; return true ;
+ if (suffix(CT, "ivas") && suffix(R2, "ivas")) {
+ CT = removeSuffix(CT, "ivas");
+ return true;
}
- if (suffix(CT,"ivos") && suffix(R2,"ivos")) {
- CT = removeSuffix(CT,"ivos") ; return true ;
+ if (suffix(CT, "ivos") && suffix(R2, "ivos")) {
+ CT = removeSuffix(CT, "ivos");
+ return true;
}
- if (suffix(CT,"iras") &&
- suffix(RV,"iras") &&
- suffixPreceded(CT,"iras","e")) {
- CT = replaceSuffix(CT,"iras","ir") ; return true ;
+ if (suffix(CT, "iras") && suffix(RV, "iras") && suffixPreceded(CT, "iras", "e")) {
+ CT = replaceSuffix(CT, "iras", "ir");
+ return true;
}
}
// suffix length = 3
if (CT.length() >= 3) {
- if (suffix(CT,"eza") && suffix(R2,"eza")) {
- CT = removeSuffix(CT,"eza") ; return true ;
+ if (suffix(CT, "eza") && suffix(R2, "eza")) {
+ CT = removeSuffix(CT, "eza");
+ return true;
}
- if (suffix(CT,"ico") && suffix(R2,"ico")) {
- CT = removeSuffix(CT,"ico") ; return true ;
+ if (suffix(CT, "ico") && suffix(R2, "ico")) {
+ CT = removeSuffix(CT, "ico");
+ return true;
}
- if (suffix(CT,"ica") && suffix(R2,"ica")) {
- CT = removeSuffix(CT,"ica") ; return true ;
+ if (suffix(CT, "ica") && suffix(R2, "ica")) {
+ CT = removeSuffix(CT, "ica");
+ return true;
}
- if (suffix(CT,"oso") && suffix(R2,"oso")) {
- CT = removeSuffix(CT,"oso") ; return true ;
+ if (suffix(CT, "oso") && suffix(R2, "oso")) {
+ CT = removeSuffix(CT, "oso");
+ return true;
}
- if (suffix(CT,"osa") && suffix(R2,"osa")) {
- CT = removeSuffix(CT,"osa") ; return true ;
+ if (suffix(CT, "osa") && suffix(R2, "osa")) {
+ CT = removeSuffix(CT, "osa");
+ return true;
}
- if (suffix(CT,"iva") && suffix(R2,"iva")) {
- CT = removeSuffix(CT,"iva") ; return true ;
+ if (suffix(CT, "iva") && suffix(R2, "iva")) {
+ CT = removeSuffix(CT, "iva");
+ return true;
}
- if (suffix(CT,"ivo") && suffix(R2,"ivo")) {
- CT = removeSuffix(CT,"ivo") ; return true ;
+ if (suffix(CT, "ivo") && suffix(R2, "ivo")) {
+ CT = removeSuffix(CT, "ivo");
+ return true;
}
- if (suffix(CT,"ira") &&
- suffix(RV,"ira") &&
- suffixPreceded(CT,"ira","e")) {
- CT = replaceSuffix(CT,"ira","ir") ; return true ;
+ if (suffix(CT, "ira") && suffix(RV, "ira") && suffixPreceded(CT, "ira", "e")) {
+ CT = replaceSuffix(CT, "ira", "ir");
+ return true;
}
}
// no ending was removed by step1
- return false ;
+ return false;
}
-
/**
* Verb suffixes.
*
- * Search for the longest among the following suffixes in RV,
- * and if found, delete.
+ * <p>Search for the longest among the following suffixes in RV, and if found, delete.
*
* @return false if no ending was removed
- */
+ */
private boolean step2() {
- if (RV == null) return false ;
+ if (RV == null) return false;
// suffix lenght = 7
if (RV.length() >= 7) {
- if (suffix(RV,"issemos")) {
- CT = removeSuffix(CT,"issemos") ; return true;
+ if (suffix(RV, "issemos")) {
+ CT = removeSuffix(CT, "issemos");
+ return true;
}
- if (suffix(RV,"essemos")) {
- CT = removeSuffix(CT,"essemos") ; return true;
+ if (suffix(RV, "essemos")) {
+ CT = removeSuffix(CT, "essemos");
+ return true;
}
- if (suffix(RV,"assemos")) {
- CT = removeSuffix(CT,"assemos") ; return true;
+ if (suffix(RV, "assemos")) {
+ CT = removeSuffix(CT, "assemos");
+ return true;
}
- if (suffix(RV,"ariamos")) {
- CT = removeSuffix(CT,"ariamos") ; return true;
+ if (suffix(RV, "ariamos")) {
+ CT = removeSuffix(CT, "ariamos");
+ return true;
}
- if (suffix(RV,"eriamos")) {
- CT = removeSuffix(CT,"eriamos") ; return true;
+ if (suffix(RV, "eriamos")) {
+ CT = removeSuffix(CT, "eriamos");
+ return true;
}
- if (suffix(RV,"iriamos")) {
- CT = removeSuffix(CT,"iriamos") ; return true;
+ if (suffix(RV, "iriamos")) {
+ CT = removeSuffix(CT, "iriamos");
+ return true;
}
}
// suffix length = 6
if (RV.length() >= 6) {
- if (suffix(RV,"iremos")) {
- CT = removeSuffix(CT,"iremos") ; return true;
+ if (suffix(RV, "iremos")) {
+ CT = removeSuffix(CT, "iremos");
+ return true;
}
- if (suffix(RV,"eremos")) {
- CT = removeSuffix(CT,"eremos") ; return true;
+ if (suffix(RV, "eremos")) {
+ CT = removeSuffix(CT, "eremos");
+ return true;
}
- if (suffix(RV,"aremos")) {
- CT = removeSuffix(CT,"aremos") ; return true;
+ if (suffix(RV, "aremos")) {
+ CT = removeSuffix(CT, "aremos");
+ return true;
}
- if (suffix(RV,"avamos")) {
- CT = removeSuffix(CT,"avamos") ; return true;
+ if (suffix(RV, "avamos")) {
+ CT = removeSuffix(CT, "avamos");
+ return true;
}
- if (suffix(RV,"iramos")) {
- CT = removeSuffix(CT,"iramos") ; return true;
+ if (suffix(RV, "iramos")) {
+ CT = removeSuffix(CT, "iramos");
+ return true;
}
- if (suffix(RV,"eramos")) {
- CT = removeSuffix(CT,"eramos") ; return true;
+ if (suffix(RV, "eramos")) {
+ CT = removeSuffix(CT, "eramos");
+ return true;
}
- if (suffix(RV,"aramos")) {
- CT = removeSuffix(CT,"aramos") ; return true;
+ if (suffix(RV, "aramos")) {
+ CT = removeSuffix(CT, "aramos");
+ return true;
}
- if (suffix(RV,"asseis")) {
- CT = removeSuffix(CT,"asseis") ; return true;
+ if (suffix(RV, "asseis")) {
+ CT = removeSuffix(CT, "asseis");
+ return true;
}
- if (suffix(RV,"esseis")) {
- CT = removeSuffix(CT,"esseis") ; return true;
+ if (suffix(RV, "esseis")) {
+ CT = removeSuffix(CT, "esseis");
+ return true;
}
- if (suffix(RV,"isseis")) {
- CT = removeSuffix(CT,"isseis") ; return true;
+ if (suffix(RV, "isseis")) {
+ CT = removeSuffix(CT, "isseis");
+ return true;
}
- if (suffix(RV,"arieis")) {
- CT = removeSuffix(CT,"arieis") ; return true;
+ if (suffix(RV, "arieis")) {
+ CT = removeSuffix(CT, "arieis");
+ return true;
}
- if (suffix(RV,"erieis")) {
- CT = removeSuffix(CT,"erieis") ; return true;
+ if (suffix(RV, "erieis")) {
+ CT = removeSuffix(CT, "erieis");
+ return true;
}
- if (suffix(RV,"irieis")) {
- CT = removeSuffix(CT,"irieis") ; return true;
+ if (suffix(RV, "irieis")) {
+ CT = removeSuffix(CT, "irieis");
+ return true;
}
}
-
// suffix length = 5
if (RV.length() >= 5) {
- if (suffix(RV,"irmos")) {
- CT = removeSuffix(CT,"irmos") ; return true;
+ if (suffix(RV, "irmos")) {
+ CT = removeSuffix(CT, "irmos");
+ return true;
}
- if (suffix(RV,"iamos")) {
- CT = removeSuffix(CT,"iamos") ; return true;
+ if (suffix(RV, "iamos")) {
+ CT = removeSuffix(CT, "iamos");
+ return true;
}
- if (suffix(RV,"armos")) {
- CT = removeSuffix(CT,"armos") ; return true;
+ if (suffix(RV, "armos")) {
+ CT = removeSuffix(CT, "armos");
+ return true;
}
- if (suffix(RV,"ermos")) {
- CT = removeSuffix(CT,"ermos") ; return true;
+ if (suffix(RV, "ermos")) {
+ CT = removeSuffix(CT, "ermos");
+ return true;
}
- if (suffix(RV,"areis")) {
- CT = removeSuffix(CT,"areis") ; return true;
+ if (suffix(RV, "areis")) {
+ CT = removeSuffix(CT, "areis");
+ return true;
}
- if (suffix(RV,"ereis")) {
- CT = removeSuffix(CT,"ereis") ; return true;
+ if (suffix(RV, "ereis")) {
+ CT = removeSuffix(CT, "ereis");
+ return true;
}
- if (suffix(RV,"ireis")) {
- CT = removeSuffix(CT,"ireis") ; return true;
+ if (suffix(RV, "ireis")) {
+ CT = removeSuffix(CT, "ireis");
+ return true;
}
- if (suffix(RV,"asses")) {
- CT = removeSuffix(CT,"asses") ; return true;
+ if (suffix(RV, "asses")) {
+ CT = removeSuffix(CT, "asses");
+ return true;
}
- if (suffix(RV,"esses")) {
- CT = removeSuffix(CT,"esses") ; return true;
+ if (suffix(RV, "esses")) {
+ CT = removeSuffix(CT, "esses");
+ return true;
}
- if (suffix(RV,"isses")) {
- CT = removeSuffix(CT,"isses") ; return true;
+ if (suffix(RV, "isses")) {
+ CT = removeSuffix(CT, "isses");
+ return true;
}
- if (suffix(RV,"astes")) {
- CT = removeSuffix(CT,"astes") ; return true;
+ if (suffix(RV, "astes")) {
+ CT = removeSuffix(CT, "astes");
+ return true;
}
- if (suffix(RV,"assem")) {
- CT = removeSuffix(CT,"assem") ; return true;
+ if (suffix(RV, "assem")) {
+ CT = removeSuffix(CT, "assem");
+ return true;
}
- if (suffix(RV,"essem")) {
- CT = removeSuffix(CT,"essem") ; return true;
+ if (suffix(RV, "essem")) {
+ CT = removeSuffix(CT, "essem");
+ return true;
}
- if (suffix(RV,"issem")) {
- CT = removeSuffix(CT,"issem") ; return true;
+ if (suffix(RV, "issem")) {
+ CT = removeSuffix(CT, "issem");
+ return true;
}
- if (suffix(RV,"ardes")) {
- CT = removeSuffix(CT,"ardes") ; return true;
+ if (suffix(RV, "ardes")) {
+ CT = removeSuffix(CT, "ardes");
+ return true;
}
- if (suffix(RV,"erdes")) {
- CT = removeSuffix(CT,"erdes") ; return true;
+ if (suffix(RV, "erdes")) {
+ CT = removeSuffix(CT, "erdes");
+ return true;
}
- if (suffix(RV,"irdes")) {
- CT = removeSuffix(CT,"irdes") ; return true;
+ if (suffix(RV, "irdes")) {
+ CT = removeSuffix(CT, "irdes");
+ return true;
}
- if (suffix(RV,"ariam")) {
- CT = removeSuffix(CT,"ariam") ; return true;
+ if (suffix(RV, "ariam")) {
+ CT = removeSuffix(CT, "ariam");
+ return true;
}
- if (suffix(RV,"eriam")) {
- CT = removeSuffix(CT,"eriam") ; return true;
+ if (suffix(RV, "eriam")) {
+ CT = removeSuffix(CT, "eriam");
+ return true;
}
- if (suffix(RV,"iriam")) {
- CT = removeSuffix(CT,"iriam") ; return true;
+ if (suffix(RV, "iriam")) {
+ CT = removeSuffix(CT, "iriam");
+ return true;
}
- if (suffix(RV,"arias")) {
- CT = removeSuffix(CT,"arias") ; return true;
+ if (suffix(RV, "arias")) {
+ CT = removeSuffix(CT, "arias");
+ return true;
}
- if (suffix(RV,"erias")) {
- CT = removeSuffix(CT,"erias") ; return true;
+ if (suffix(RV, "erias")) {
+ CT = removeSuffix(CT, "erias");
+ return true;
}
- if (suffix(RV,"irias")) {
- CT = removeSuffix(CT,"irias") ; return true;
+ if (suffix(RV, "irias")) {
+ CT = removeSuffix(CT, "irias");
+ return true;
}
- if (suffix(RV,"estes")) {
- CT = removeSuffix(CT,"estes") ; return true;
+ if (suffix(RV, "estes")) {
+ CT = removeSuffix(CT, "estes");
+ return true;
}
- if (suffix(RV,"istes")) {
- CT = removeSuffix(CT,"istes") ; return true;
+ if (suffix(RV, "istes")) {
+ CT = removeSuffix(CT, "istes");
+ return true;
}
- if (suffix(RV,"areis")) {
- CT = removeSuffix(CT,"areis") ; return true;
+ if (suffix(RV, "areis")) {
+ CT = removeSuffix(CT, "areis");
+ return true;
}
- if (suffix(RV,"aveis")) {
- CT = removeSuffix(CT,"aveis") ; return true;
+ if (suffix(RV, "aveis")) {
+ CT = removeSuffix(CT, "aveis");
+ return true;
}
}
// suffix length = 4
if (RV.length() >= 4) {
- if (suffix(RV,"aria")) {
- CT = removeSuffix(CT,"aria") ; return true;
+ if (suffix(RV, "aria")) {
+ CT = removeSuffix(CT, "aria");
+ return true;
}
- if (suffix(RV,"eria")) {
- CT = removeSuffix(CT,"eria") ; return true;
+ if (suffix(RV, "eria")) {
+ CT = removeSuffix(CT, "eria");
+ return true;
}
- if (suffix(RV,"iria")) {
- CT = removeSuffix(CT,"iria") ; return true;
+ if (suffix(RV, "iria")) {
+ CT = removeSuffix(CT, "iria");
+ return true;
}
- if (suffix(RV,"asse")) {
- CT = removeSuffix(CT,"asse") ; return true;
+ if (suffix(RV, "asse")) {
+ CT = removeSuffix(CT, "asse");
+ return true;
}
- if (suffix(RV,"esse")) {
- CT = removeSuffix(CT,"esse") ; return true;
+ if (suffix(RV, "esse")) {
+ CT = removeSuffix(CT, "esse");
+ return true;
}
- if (suffix(RV,"isse")) {
- CT = removeSuffix(CT,"isse") ; return true;
+ if (suffix(RV, "isse")) {
+ CT = removeSuffix(CT, "isse");
+ return true;
}
- if (suffix(RV,"aste")) {
- CT = removeSuffix(CT,"aste") ; return true;
+ if (suffix(RV, "aste")) {
+ CT = removeSuffix(CT, "aste");
+ return true;
}
- if (suffix(RV,"este")) {
- CT = removeSuffix(CT,"este") ; return true;
+ if (suffix(RV, "este")) {
+ CT = removeSuffix(CT, "este");
+ return true;
}
- if (suffix(RV,"iste")) {
- CT = removeSuffix(CT,"iste") ; return true;
+ if (suffix(RV, "iste")) {
+ CT = removeSuffix(CT, "iste");
+ return true;
}
- if (suffix(RV,"arei")) {
- CT = removeSuffix(CT,"arei") ; return true;
+ if (suffix(RV, "arei")) {
+ CT = removeSuffix(CT, "arei");
+ return true;
}
- if (suffix(RV,"erei")) {
- CT = removeSuffix(CT,"erei") ; return true;
+ if (suffix(RV, "erei")) {
+ CT = removeSuffix(CT, "erei");
+ return true;
}
- if (suffix(RV,"irei")) {
- CT = removeSuffix(CT,"irei") ; return true;
+ if (suffix(RV, "irei")) {
+ CT = removeSuffix(CT, "irei");
+ return true;
}
- if (suffix(RV,"aram")) {
- CT = removeSuffix(CT,"aram") ; return true;
+ if (suffix(RV, "aram")) {
+ CT = removeSuffix(CT, "aram");
+ return true;
}
- if (suffix(RV,"eram")) {
- CT = removeSuffix(CT,"eram") ; return true;
+ if (suffix(RV, "eram")) {
+ CT = removeSuffix(CT, "eram");
+ return true;
}
- if (suffix(RV,"iram")) {
- CT = removeSuffix(CT,"iram") ; return true;
+ if (suffix(RV, "iram")) {
+ CT = removeSuffix(CT, "iram");
+ return true;
}
- if (suffix(RV,"avam")) {
- CT = removeSuffix(CT,"avam") ; return true;
+ if (suffix(RV, "avam")) {
+ CT = removeSuffix(CT, "avam");
+ return true;
}
- if (suffix(RV,"arem")) {
- CT = removeSuffix(CT,"arem") ; return true;
+ if (suffix(RV, "arem")) {
+ CT = removeSuffix(CT, "arem");
+ return true;
}
- if (suffix(RV,"erem")) {
- CT = removeSuffix(CT,"erem") ; return true;
+ if (suffix(RV, "erem")) {
+ CT = removeSuffix(CT, "erem");
+ return true;
}
- if (suffix(RV,"irem")) {
- CT = removeSuffix(CT,"irem") ; return true;
+ if (suffix(RV, "irem")) {
+ CT = removeSuffix(CT, "irem");
+ return true;
}
- if (suffix(RV,"ando")) {
- CT = removeSuffix(CT,"ando") ; return true;
+ if (suffix(RV, "ando")) {
+ CT = removeSuffix(CT, "ando");
+ return true;
}
- if (suffix(RV,"endo")) {
- CT = removeSuffix(CT,"endo") ; return true;
+ if (suffix(RV, "endo")) {
+ CT = removeSuffix(CT, "endo");
+ return true;
}
- if (suffix(RV,"indo")) {
- CT = removeSuffix(CT,"indo") ; return true;
+ if (suffix(RV, "indo")) {
+ CT = removeSuffix(CT, "indo");
+ return true;
}
- if (suffix(RV,"arao")) {
- CT = removeSuffix(CT,"arao") ; return true;
+ if (suffix(RV, "arao")) {
+ CT = removeSuffix(CT, "arao");
+ return true;
}
- if (suffix(RV,"erao")) {
- CT = removeSuffix(CT,"erao") ; return true;
+ if (suffix(RV, "erao")) {
+ CT = removeSuffix(CT, "erao");
+ return true;
}
- if (suffix(RV,"irao")) {
- CT = removeSuffix(CT,"irao") ; return true;
+ if (suffix(RV, "irao")) {
+ CT = removeSuffix(CT, "irao");
+ return true;
}
- if (suffix(RV,"adas")) {
- CT = removeSuffix(CT,"adas") ; return true;
+ if (suffix(RV, "adas")) {
+ CT = removeSuffix(CT, "adas");
+ return true;
}
- if (suffix(RV,"idas")) {
- CT = removeSuffix(CT,"idas") ; return true;
+ if (suffix(RV, "idas")) {
+ CT = removeSuffix(CT, "idas");
+ return true;
}
- if (suffix(RV,"aras")) {
- CT = removeSuffix(CT,"aras") ; return true;
+ if (suffix(RV, "aras")) {
+ CT = removeSuffix(CT, "aras");
+ return true;
}
- if (suffix(RV,"eras")) {
- CT = removeSuffix(CT,"eras") ; return true;
+ if (suffix(RV, "eras")) {
+ CT = removeSuffix(CT, "eras");
+ return true;
}
- if (suffix(RV,"iras")) {
- CT = removeSuffix(CT,"iras") ; return true;
+ if (suffix(RV, "iras")) {
+ CT = removeSuffix(CT, "iras");
+ return true;
}
- if (suffix(RV,"avas")) {
- CT = removeSuffix(CT,"avas") ; return true;
+ if (suffix(RV, "avas")) {
+ CT = removeSuffix(CT, "avas");
+ return true;
}
- if (suffix(RV,"ares")) {
- CT = removeSuffix(CT,"ares") ; return true;
+ if (suffix(RV, "ares")) {
+ CT = removeSuffix(CT, "ares");
+ return true;
}
- if (suffix(RV,"eres")) {
- CT = removeSuffix(CT,"eres") ; return true;
+ if (suffix(RV, "eres")) {
+ CT = removeSuffix(CT, "eres");
+ return true;
}
- if (suffix(RV,"ires")) {
- CT = removeSuffix(CT,"ires") ; return true;
+ if (suffix(RV, "ires")) {
+ CT = removeSuffix(CT, "ires");
+ return true;
}
- if (suffix(RV,"ados")) {
- CT = removeSuffix(CT,"ados") ; return true;
+ if (suffix(RV, "ados")) {
+ CT = removeSuffix(CT, "ados");
+ return true;
}
- if (suffix(RV,"idos")) {
- CT = removeSuffix(CT,"idos") ; return true;
+ if (suffix(RV, "idos")) {
+ CT = removeSuffix(CT, "idos");
+ return true;
}
- if (suffix(RV,"amos")) {
- CT = removeSuffix(CT,"amos") ; return true;
+ if (suffix(RV, "amos")) {
+ CT = removeSuffix(CT, "amos");
+ return true;
}
- if (suffix(RV,"emos")) {
- CT = removeSuffix(CT,"emos") ; return true;
+ if (suffix(RV, "emos")) {
+ CT = removeSuffix(CT, "emos");
+ return true;
}
- if (suffix(RV,"imos")) {
- CT = removeSuffix(CT,"imos") ; return true;
+ if (suffix(RV, "imos")) {
+ CT = removeSuffix(CT, "imos");
+ return true;
}
- if (suffix(RV,"iras")) {
- CT = removeSuffix(CT,"iras") ; return true;
+ if (suffix(RV, "iras")) {
+ CT = removeSuffix(CT, "iras");
+ return true;
}
- if (suffix(RV,"ieis")) {
- CT = removeSuffix(CT,"ieis") ; return true;
+ if (suffix(RV, "ieis")) {
+ CT = removeSuffix(CT, "ieis");
+ return true;
}
}
// suffix length = 3
if (RV.length() >= 3) {
- if (suffix(RV,"ada")) {
- CT = removeSuffix(CT,"ada") ; return true;
+ if (suffix(RV, "ada")) {
+ CT = removeSuffix(CT, "ada");
+ return true;
}
- if (suffix(RV,"ida")) {
- CT = removeSuffix(CT,"ida") ; return true;
+ if (suffix(RV, "ida")) {
+ CT = removeSuffix(CT, "ida");
+ return true;
}
- if (suffix(RV,"ara")) {
- CT = removeSuffix(CT,"ara") ; return true;
+ if (suffix(RV, "ara")) {
+ CT = removeSuffix(CT, "ara");
+ return true;
}
- if (suffix(RV,"era")) {
- CT = removeSuffix(CT,"era") ; return true;
+ if (suffix(RV, "era")) {
+ CT = removeSuffix(CT, "era");
+ return true;
}
- if (suffix(RV,"ira")) {
- CT = removeSuffix(CT,"ava") ; return true;
+ if (suffix(RV, "ira")) {
+ CT = removeSuffix(CT, "ava");
+ return true;
}
- if (suffix(RV,"iam")) {
- CT = removeSuffix(CT,"iam") ; return true;
+ if (suffix(RV, "iam")) {
+ CT = removeSuffix(CT, "iam");
+ return true;
}
- if (suffix(RV,"ado")) {
- CT = removeSuffix(CT,"ado") ; return true;
+ if (suffix(RV, "ado")) {
+ CT = removeSuffix(CT, "ado");
+ return true;
}
- if (suffix(RV,"ido")) {
- CT = removeSuffix(CT,"ido") ; return true;
+ if (suffix(RV, "ido")) {
+ CT = removeSuffix(CT, "ido");
+ return true;
}
- if (suffix(RV,"ias")) {
- CT = removeSuffix(CT,"ias") ; return true;
+ if (suffix(RV, "ias")) {
+ CT = removeSuffix(CT, "ias");
+ return true;
}
- if (suffix(RV,"ais")) {
- CT = removeSuffix(CT,"ais") ; return true;
+ if (suffix(RV, "ais")) {
+ CT = removeSuffix(CT, "ais");
+ return true;
}
- if (suffix(RV,"eis")) {
- CT = removeSuffix(CT,"eis") ; return true;
+ if (suffix(RV, "eis")) {
+ CT = removeSuffix(CT, "eis");
+ return true;
}
- if (suffix(RV,"ira")) {
- CT = removeSuffix(CT,"ira") ; return true;
+ if (suffix(RV, "ira")) {
+ CT = removeSuffix(CT, "ira");
+ return true;
}
- if (suffix(RV,"ear")) {
- CT = removeSuffix(CT,"ear") ; return true;
+ if (suffix(RV, "ear")) {
+ CT = removeSuffix(CT, "ear");
+ return true;
}
}
// suffix length = 2
if (RV.length() >= 2) {
- if (suffix(RV,"ia")) {
- CT = removeSuffix(CT,"ia") ; return true;
+ if (suffix(RV, "ia")) {
+ CT = removeSuffix(CT, "ia");
+ return true;
}
- if (suffix(RV,"ei")) {
- CT = removeSuffix(CT,"ei") ; return true;
+ if (suffix(RV, "ei")) {
+ CT = removeSuffix(CT, "ei");
+ return true;
}
- if (suffix(RV,"am")) {
- CT = removeSuffix(CT,"am") ; return true;
+ if (suffix(RV, "am")) {
+ CT = removeSuffix(CT, "am");
+ return true;
}
- if (suffix(RV,"em")) {
- CT = removeSuffix(CT,"em") ; return true;
+ if (suffix(RV, "em")) {
+ CT = removeSuffix(CT, "em");
+ return true;
}
- if (suffix(RV,"ar")) {
- CT = removeSuffix(CT,"ar") ; return true;
+ if (suffix(RV, "ar")) {
+ CT = removeSuffix(CT, "ar");
+ return true;
}
- if (suffix(RV,"er")) {
- CT = removeSuffix(CT,"er") ; return true;
+ if (suffix(RV, "er")) {
+ CT = removeSuffix(CT, "er");
+ return true;
}
- if (suffix(RV,"ir")) {
- CT = removeSuffix(CT,"ir") ; return true;
+ if (suffix(RV, "ir")) {
+ CT = removeSuffix(CT, "ir");
+ return true;
}
- if (suffix(RV,"as")) {
- CT = removeSuffix(CT,"as") ; return true;
+ if (suffix(RV, "as")) {
+ CT = removeSuffix(CT, "as");
+ return true;
}
- if (suffix(RV,"es")) {
- CT = removeSuffix(CT,"es") ; return true;
+ if (suffix(RV, "es")) {
+ CT = removeSuffix(CT, "es");
+ return true;
}
- if (suffix(RV,"is")) {
- CT = removeSuffix(CT,"is") ; return true;
+ if (suffix(RV, "is")) {
+ CT = removeSuffix(CT, "is");
+ return true;
}
- if (suffix(RV,"eu")) {
- CT = removeSuffix(CT,"eu") ; return true;
+ if (suffix(RV, "eu")) {
+ CT = removeSuffix(CT, "eu");
+ return true;
}
- if (suffix(RV,"iu")) {
- CT = removeSuffix(CT,"iu") ; return true;
+ if (suffix(RV, "iu")) {
+ CT = removeSuffix(CT, "iu");
+ return true;
}
- if (suffix(RV,"iu")) {
- CT = removeSuffix(CT,"iu") ; return true;
+ if (suffix(RV, "iu")) {
+ CT = removeSuffix(CT, "iu");
+ return true;
}
- if (suffix(RV,"ou")) {
- CT = removeSuffix(CT,"ou") ; return true;
+ if (suffix(RV, "ou")) {
+ CT = removeSuffix(CT, "ou");
+ return true;
}
}
// no ending was removed by step2
- return false ;
+ return false;
}
- /**
- * Delete suffix 'i' if in RV and preceded by 'c'
- *
- */
+ /** Delete suffix 'i' if in RV and preceded by 'c' */
private void step3() {
- if (RV == null) return ;
+ if (RV == null) return;
- if (suffix(RV,"i") && suffixPreceded(RV,"i","c")) {
- CT = removeSuffix(CT,"i") ;
+ if (suffix(RV, "i") && suffixPreceded(RV, "i", "c")) {
+ CT = removeSuffix(CT, "i");
}
-
}
/**
* Residual suffix
*
- * If the word ends with one of the suffixes (os a i o á í ó)
- * in RV, delete it
- *
- */
+ * <p>If the word ends with one of the suffixes (os a i o á í ó) in RV, delete it
+ */
private void step4() {
- if (RV == null) return ;
+ if (RV == null) return;
- if (suffix(RV,"os")) {
- CT = removeSuffix(CT,"os") ; return ;
+ if (suffix(RV, "os")) {
+ CT = removeSuffix(CT, "os");
+ return;
}
- if (suffix(RV,"a")) {
- CT = removeSuffix(CT,"a") ; return ;
+ if (suffix(RV, "a")) {
+ CT = removeSuffix(CT, "a");
+ return;
}
- if (suffix(RV,"i")) {
- CT = removeSuffix(CT,"i") ; return ;
+ if (suffix(RV, "i")) {
+ CT = removeSuffix(CT, "i");
+ return;
}
- if (suffix(RV,"o")) {
- CT = removeSuffix(CT,"o") ; return ;
+ if (suffix(RV, "o")) {
+ CT = removeSuffix(CT, "o");
+ return;
}
-
}
/**
- * If the word ends with one of ( e é ê) in RV,delete it,
- * and if preceded by 'gu' (or 'ci') with the 'u' (or 'i') in RV,
- * delete the 'u' (or 'i')
- *
- * Or if the word ends ç remove the cedilha
+ * If the word ends with one of ( e é ê) in RV,delete it, and if preceded by 'gu' (or 'ci') with
+ * the 'u' (or 'i') in RV, delete the 'u' (or 'i')
*
- */
+ * <p>Or if the word ends ç remove the cedilha
+ */
private void step5() {
- if (RV == null) return ;
+ if (RV == null) return;
- if (suffix(RV,"e")) {
- if (suffixPreceded(RV,"e","gu")) {
- CT = removeSuffix(CT,"e") ;
- CT = removeSuffix(CT,"u") ;
- return ;
+ if (suffix(RV, "e")) {
+ if (suffixPreceded(RV, "e", "gu")) {
+ CT = removeSuffix(CT, "e");
+ CT = removeSuffix(CT, "u");
+ return;
}
- if (suffixPreceded(RV,"e","ci")) {
- CT = removeSuffix(CT,"e") ;
- CT = removeSuffix(CT,"i") ;
- return ;
+ if (suffixPreceded(RV, "e", "ci")) {
+ CT = removeSuffix(CT, "e");
+ CT = removeSuffix(CT, "i");
+ return;
}
- CT = removeSuffix(CT,"e") ; return ;
+ CT = removeSuffix(CT, "e");
+ return;
}
}
/**
* For log and debug purpose
*
- * @return TERM, CT, RV, R1 and R2
+ * @return TERM, CT, RV, R1 and R2
*/
public String log() {
- return " (TERM = " + TERM + ")" +
- " (CT = " + CT +")" +
- " (RV = " + RV +")" +
- " (R1 = " + R1 +")" +
- " (R2 = " + R2 +")" ;
+ return " (TERM = "
+ + TERM
+ + ")"
+ + " (CT = "
+ + CT
+ + ")"
+ + " (RV = "
+ + RV
+ + ")"
+ + " (R1 = "
+ + R1
+ + ")"
+ + " (R2 = "
+ + R2
+ + ")";
}
-
}
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/package-info.java
index 080389b..a7ef132 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analyzer for Brazilian Portuguese.
- */
+/** Analyzer for Brazilian Portuguese. */
package org.apache.lucene.analysis.br;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java
index 8d90e95..a1f71fe 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java
@@ -16,11 +16,9 @@
*/
package org.apache.lucene.analysis.ca;
-
import java.io.IOException;
import java.io.Reader;
import java.util.Arrays;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -41,35 +39,34 @@ import org.tartarus.snowball.ext.CatalanStemmer;
*/
public final class CatalanAnalyzer extends StopwordAnalyzerBase {
private final CharArraySet stemExclusionSet;
-
+
/** File containing default Catalan stopwords. */
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
-
- private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
- new CharArraySet(
- Arrays.asList(
- "d", "l", "m", "n", "s", "t"
- ), true));
-
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
+ private static final CharArraySet DEFAULT_ARTICLES =
+ CharArraySet.unmodifiableSet(
+ new CharArraySet(Arrays.asList("d", "l", "m", "n", "s", "t"), true));
+
/**
* Returns an unmodifiable instance of the default stop words set.
+ *
* @return default stop words set.
*/
- public static CharArraySet getDefaultStopSet(){
+ public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
/**
- * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
- * accesses the static final set the first time.;
+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the
+ * static final set the first time.;
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
static {
try {
- DEFAULT_STOP_SET = loadStopwordSet(false,
- CatalanAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
+ DEFAULT_STOP_SET =
+ loadStopwordSet(false, CatalanAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@@ -78,16 +75,14 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
}
}
- /**
- * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
- */
+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
public CatalanAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
-
+
/**
* Builds an analyzer with the given stop words.
- *
+ *
* @param stopwords a stopword set
*/
public CatalanAnalyzer(CharArraySet stopwords) {
@@ -95,10 +90,9 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
}
/**
- * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is
- * provided this analyzer will add a {@link SetKeywordMarkerFilter} before
- * stemming.
- *
+ * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is provided
+ * this analyzer will add a {@link SetKeywordMarkerFilter} before stemming.
+ *
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
@@ -108,16 +102,13 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
}
/**
- * Creates a
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * which tokenizes all the text in the provided {@link Reader}.
- *
- * @return A
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from an {@link StandardTokenizer} filtered with
- * {@link ElisionFilter}, {@link LowerCaseFilter},
- * {@link StopFilter}, {@link SetKeywordMarkerFilter} if a stem exclusion set is
- * provided and {@link SnowballFilter}.
+ * Creates a {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} which tokenizes all
+ * the text in the provided {@link Reader}.
+ *
+ * @return A {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from an
+ * {@link StandardTokenizer} filtered with {@link ElisionFilter}, {@link LowerCaseFilter},
+ * {@link StopFilter}, {@link SetKeywordMarkerFilter} if a stem exclusion set is provided and
+ * {@link SnowballFilter}.
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
@@ -125,8 +116,9 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
TokenStream result = new ElisionFilter(source, DEFAULT_ARTICLES);
result = new LowerCaseFilter(result);
result = new StopFilter(result, stopwords);
- if(!stemExclusionSet.isEmpty())
+ if (!stemExclusionSet.isEmpty()) {
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+ }
result = new SnowballFilter(result, new CatalanStemmer());
return new TokenStreamComponents(source, result);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/package-info.java
index 7f8b0da..312e683 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analyzer for Catalan.
- */
+/** Analyzer for Catalan. */
package org.apache.lucene.analysis.ca;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java
index 4fba9fe..b9ce02b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java
@@ -16,24 +16,22 @@
*/
package org.apache.lucene.analysis.charfilter;
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.util.ArrayUtil;
-
import java.io.Reader;
import java.util.Arrays;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.util.ArrayUtil;
/**
- * Base utility class for implementing a {@link CharFilter}.
- * You subclass this, and then record mappings by calling
- * {@link #addOffCorrectMap}, and then invoke the correct
- * method to correct an offset.
+ * Base utility class for implementing a {@link CharFilter}. You subclass this, and then record
+ * mappings by calling {@link #addOffCorrectMap}, and then invoke the correct method to correct an
+ * offset.
*/
public abstract class BaseCharFilter extends CharFilter {
private int offsets[];
private int diffs[];
private int size = 0;
-
+
public BaseCharFilter(Reader in) {
super(in);
}
@@ -53,24 +51,19 @@ public abstract class BaseCharFilter extends CharFilter {
final int diff = index < 0 ? 0 : diffs[index];
return currentOff + diff;
}
-
+
protected int getLastCumulativeDiff() {
- return offsets == null ?
- 0 : diffs[size-1];
+ return offsets == null ? 0 : diffs[size - 1];
}
/**
- * <p>
- * Adds an offset correction mapping at the given output stream offset.
- * </p>
- * <p>
- * Assumption: the offset given with each successive call to this method
- * will not be smaller than the offset given at the previous invocation.
- * </p>
+ * Adds an offset correction mapping at the given output stream offset.
+ *
+ * <p>Assumption: the offset given with each successive call to this method will not be smaller
+ * than the offset given at the previous invocation.
*
* @param off The output stream offset at which to apply the correction
- * @param cumulativeDiff The input offset is given by adding this
- * to the output offset
+ * @param cumulativeDiff The input offset is given by adding this to the output offset
*/
protected void addOffCorrectMap(int off, int cumulativeDiff) {
if (offsets == null) {
@@ -80,11 +73,19 @@ public abstract class BaseCharFilter extends CharFilter {
offsets = ArrayUtil.grow(offsets);
diffs = ArrayUtil.grow(diffs);
}
-
+
assert (size == 0 || off >= offsets[size - 1])
- : "Offset #" + size + "(" + off + ") is less than the last recorded offset "
- + offsets[size - 1] + "\n" + Arrays.toString(offsets) + "\n" + Arrays.toString(diffs);
-
+ : "Offset #"
+ + size
+ + "("
+ + off
+ + ") is less than the last recorded offset "
+ + offsets[size - 1]
+ + "\n"
+ + Arrays.toString(offsets)
+ + "\n"
+ + Arrays.toString(diffs);
+
if (size == 0 || off != offsets[size - 1]) {
offsets[size] = off;
diffs[size++] = cumulativeDiff;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterFactory.java
index 1712227..84fc1cf 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterFactory.java
@@ -16,16 +16,15 @@
*/
package org.apache.lucene.analysis.charfilter;
-
-import org.apache.lucene.analysis.CharFilterFactory;
-
import java.io.Reader;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.lucene.analysis.CharFilterFactory;
/**
- * Factory for {@link HTMLStripCharFilter}.
+ * Factory for {@link HTMLStripCharFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_html" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -44,9 +43,9 @@ public class HTMLStripCharFilterFactory extends CharFilterFactory {
final Set<String> escapedTags;
static final Pattern TAG_NAME_PATTERN = Pattern.compile("[^\\s,]+");
-
+
/** Creates a new HTMLStripCharFilterFactory */
- public HTMLStripCharFilterFactory(Map<String,String> args) {
+ public HTMLStripCharFilterFactory(Map<String, String> args) {
super(args);
escapedTags = getSet(args, "escapedTags");
if (!args.isEmpty()) {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
index 5fffda9..04b7368 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.charfilter;
import java.io.IOException;
import java.io.Reader;
import java.util.Map;
-
import org.apache.lucene.analysis.CharFilter; // javadocs
import org.apache.lucene.analysis.util.RollingCharBuffer;
import org.apache.lucene.util.CharsRef;
@@ -28,14 +27,10 @@ import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.Outputs;
/**
- * Simplistic {@link CharFilter} that applies the mappings
- * contained in a {@link NormalizeCharMap} to the character
- * stream, and correcting the resulting changes to the
- * offsets. Matching is greedy (longest pattern matching at
- * a given point wins). Replacement is allowed to be the
- * empty string.
+ * Simplistic {@link CharFilter} that applies the mappings contained in a {@link NormalizeCharMap}
+ * to the character stream, and correcting the resulting changes to the offsets. Matching is greedy
+ * (longest pattern matching at a given point wins). Replacement is allowed to be the empty string.
*/
-
public class MappingCharFilter extends BaseCharFilter {
private final Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
@@ -43,7 +38,7 @@ public class MappingCharFilter extends BaseCharFilter {
private final FST.BytesReader fstReader;
private final RollingCharBuffer buffer = new RollingCharBuffer();
private final FST.Arc<CharsRef> scratchArc = new FST.Arc<>();
- private final Map<Character,FST.Arc<CharsRef>> cachedRootArcs;
+ private final Map<Character, FST.Arc<CharsRef>> cachedRootArcs;
private CharsRef replacement;
private int replacementPointer;
@@ -75,11 +70,12 @@ public class MappingCharFilter extends BaseCharFilter {
@Override
public int read() throws IOException {
- //System.out.println("\nread");
- while(true) {
+ // System.out.println("\nread");
+ while (true) {
if (replacement != null && replacementPointer < replacement.length) {
- //System.out.println(" return repl[" + replacementPointer + "]=" + replacement.chars[replacement.offset + replacementPointer]);
+ // System.out.println(" return repl[" + replacementPointer + "]=" +
+ // replacement.chars[replacement.offset + replacementPointer]);
return replacement.chars[replacement.offset + replacementPointer++];
}
@@ -141,7 +137,7 @@ public class MappingCharFilter extends BaseCharFilter {
if (lastMatch != null) {
inputOff += lastMatchLen;
- //System.out.println(" match! len=" + lastMatchLen + " repl=" + lastMatch);
+ // System.out.println(" match! len=" + lastMatchLen + " repl=" + lastMatch);
final int diff = lastMatchLen - lastMatch.length;
@@ -155,7 +151,7 @@ public class MappingCharFilter extends BaseCharFilter {
// the "extra" chars all back to the same input
// offset:
final int outputStart = inputOff - prevCumulativeDiff;
- for(int extraIDX=0;extraIDX<-diff;extraIDX++) {
+ for (int extraIDX = 0; extraIDX < -diff; extraIDX++) {
addOffCorrectMap(outputStart + extraIDX, prevCumulativeDiff - extraIDX - 1);
}
}
@@ -178,7 +174,7 @@ public class MappingCharFilter extends BaseCharFilter {
@Override
public int read(char[] cbuf, int off, int len) throws IOException {
int numRead = 0;
- for(int i = off; i < off + len; i++) {
+ for (int i = off; i < off + len; i++) {
int c = read();
if (c == -1) break;
cbuf[i] = (char) c;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
index 8296719..fa77a18 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.analysis.charfilter;
-
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
@@ -24,13 +23,13 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import org.apache.lucene.analysis.CharFilterFactory;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
/**
- * Factory for {@link MappingCharFilter}.
+ * Factory for {@link MappingCharFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_map" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -42,8 +41,7 @@ import org.apache.lucene.util.ResourceLoaderAware;
* @since Solr 1.4
* @lucene.spi {@value #NAME}
*/
-public class MappingCharFilterFactory extends CharFilterFactory implements
- ResourceLoaderAware {
+public class MappingCharFilterFactory extends CharFilterFactory implements ResourceLoaderAware {
/** SPI name */
public static final String NAME = "mapping";
@@ -52,7 +50,7 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
private final String mapping;
/** Creates a new MappingCharFilterFactory */
- public MappingCharFilterFactory(Map<String,String> args) {
+ public MappingCharFilterFactory(Map<String, String> args) {
super(args);
mapping = get(args, "mapping");
if (!args.isEmpty()) {
@@ -90,7 +88,7 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
public Reader create(Reader input) {
// if the map is null, it means there's actually no mappings... just return the original stream
// as there is nothing to do here.
- return normMap == null ? input : new MappingCharFilter(normMap,input);
+ return normMap == null ? input : new MappingCharFilter(normMap, input);
}
@Override
@@ -99,48 +97,62 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
}
// "source" => "target"
- static Pattern p = Pattern.compile( "\"(.*)\"\\s*=>\\s*\"(.*)\"\\s*$" );
+ static Pattern p = Pattern.compile("\"(.*)\"\\s*=>\\s*\"(.*)\"\\s*$");
- protected void parseRules( List<String> rules, NormalizeCharMap.Builder builder ){
- for( String rule : rules ){
- Matcher m = p.matcher( rule );
- if( !m.find() )
- throw new IllegalArgumentException("Invalid Mapping Rule : [" + rule + "], file = " + mapping);
- builder.add( parseString( m.group( 1 ) ), parseString( m.group( 2 ) ) );
+ protected void parseRules(List<String> rules, NormalizeCharMap.Builder builder) {
+ for (String rule : rules) {
+ Matcher m = p.matcher(rule);
+ if (!m.find())
+ throw new IllegalArgumentException(
+ "Invalid Mapping Rule : [" + rule + "], file = " + mapping);
+ builder.add(parseString(m.group(1)), parseString(m.group(2)));
}
}
char[] out = new char[256];
-
- protected String parseString( String s ){
+
+ protected String parseString(String s) {
int readPos = 0;
int len = s.length();
int writePos = 0;
- while( readPos < len ){
- char c = s.charAt( readPos++ );
- if( c == '\\' ){
- if( readPos >= len )
+ while (readPos < len) {
+ char c = s.charAt(readPos++);
+ if (c == '\\') {
+ if (readPos >= len)
throw new IllegalArgumentException("Invalid escaped char in [" + s + "]");
- c = s.charAt( readPos++ );
- switch( c ) {
- case '\\' : c = '\\'; break;
- case '"' : c = '"'; break;
- case 'n' : c = '\n'; break;
- case 't' : c = '\t'; break;
- case 'r' : c = '\r'; break;
- case 'b' : c = '\b'; break;
- case 'f' : c = '\f'; break;
- case 'u' :
- if( readPos + 3 >= len )
+ c = s.charAt(readPos++);
+ switch (c) {
+ case '\\':
+ c = '\\';
+ break;
+ case '"':
+ c = '"';
+ break;
+ case 'n':
+ c = '\n';
+ break;
+ case 't':
+ c = '\t';
+ break;
+ case 'r':
+ c = '\r';
+ break;
+ case 'b':
+ c = '\b';
+ break;
+ case 'f':
+ c = '\f';
+ break;
+ case 'u':
+ if (readPos + 3 >= len)
throw new IllegalArgumentException("Invalid escaped char in [" + s + "]");
- c = (char)Integer.parseInt( s.substring( readPos, readPos + 4 ), 16 );
+ c = (char) Integer.parseInt(s.substring(readPos, readPos + 4), 16);
readPos += 4;
break;
}
}
out[writePos++] = c;
}
- return new String( out, 0, writePos );
+ return new String(out, 0, writePos);
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
index a217b4c..b3dba24 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
-
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.CharSequenceOutputs;
@@ -32,14 +31,13 @@ import org.apache.lucene.util.fst.Util;
// TODO: save/load?
/**
- * Holds a map of String input to String output, to be used
- * with {@link MappingCharFilter}. Use the {@link Builder}
- * to create this.
+ * Holds a map of String input to String output, to be used with {@link MappingCharFilter}. Use the
+ * {@link Builder} to create this.
*/
public class NormalizeCharMap {
final FST<CharsRef> map;
- final Map<Character,FST.Arc<CharsRef>> cachedRootArcs = new HashMap<>();
+ final Map<Character, FST.Arc<CharsRef>> cachedRootArcs = new HashMap<>();
// Use the builder to create:
private NormalizeCharMap(FST<CharsRef> map) {
@@ -52,16 +50,18 @@ public class NormalizeCharMap {
map.getFirstArc(scratchArc);
if (FST.targetHasArcs(scratchArc)) {
map.readFirstRealTargetArc(scratchArc.target(), scratchArc, fstReader);
- while(true) {
+ while (true) {
assert scratchArc.label() != FST.END_LABEL;
- cachedRootArcs.put(Character.valueOf((char) scratchArc.label()), new FST.Arc<CharsRef>().copyFrom(scratchArc));
+ cachedRootArcs.put(
+ Character.valueOf((char) scratchArc.label()),
+ new FST.Arc<CharsRef>().copyFrom(scratchArc));
if (scratchArc.isLast()) {
break;
}
map.readNextRealArc(scratchArc, fstReader);
}
}
- //System.out.println("cached " + cachedRootArcs.size() + " root arcs");
+ // System.out.println("cached " + cachedRootArcs.size() + " root arcs");
} catch (IOException ioe) {
// Bogus FST IOExceptions!! (will never happen)
throw new RuntimeException(ioe);
@@ -71,27 +71,27 @@ public class NormalizeCharMap {
/**
* Builds an NormalizeCharMap.
- * <p>
- * Call add() until you have added all the mappings, then call build() to get a NormalizeCharMap
+ *
+ * <p>Call add() until you have added all the mappings, then call build() to get a
+ * NormalizeCharMap
+ *
* @lucene.experimental
*/
public static class Builder {
- private final Map<String,String> pendingPairs = new TreeMap<>();
+ private final Map<String, String> pendingPairs = new TreeMap<>();
- /** Records a replacement to be applied to the input
- * stream. Whenever <code>singleMatch</code> occurs in
- * the input, it will be replaced with
- * <code>replacement</code>.
+ /**
+ * Records a replacement to be applied to the input stream. Whenever <code>singleMatch</code>
+ * occurs in the input, it will be replaced with <code>replacement</code>.
*
* @param match input String to be replaced
* @param replacement output String
- * @throws IllegalArgumentException if
- * <code>match</code> is the empty string, or was
- * already previously added
+ * @throws IllegalArgumentException if <code>match</code> is the empty string, or was already
+ * previously added
*/
public void add(String match, String replacement) {
- if (match.length() == 0 ){
+ if (match.length() == 0) {
throw new IllegalArgumentException("cannot match the empty string");
}
if (pendingPairs.containsKey(match)) {
@@ -100,8 +100,7 @@ public class NormalizeCharMap {
pendingPairs.put(match, replacement);
}
- /** Builds the NormalizeCharMap; call this once you
- * are done calling {@link #add}. */
+ /** Builds the NormalizeCharMap; call this once you are done calling {@link #add}. */
public NormalizeCharMap build() {
final FST<CharsRef> map;
@@ -109,9 +108,8 @@ public class NormalizeCharMap {
final Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
final FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
final IntsRefBuilder scratch = new IntsRefBuilder();
- for(Map.Entry<String,String> ent : pendingPairs.entrySet()) {
- fstCompiler.add(Util.toUTF16(ent.getKey(), scratch),
- new CharsRef(ent.getValue()));
+ for (Map.Entry<String, String> ent : pendingPairs.entrySet()) {
+ fstCompiler.add(Util.toUTF16(ent.getKey(), scratch), new CharsRef(ent.getValue()));
}
map = fstCompiler.compile();
pendingPairs.clear();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/package-info.java
index 12a43a5..54a24d3 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/package-info.java
@@ -17,42 +17,31 @@
/**
* Normalization of text before the tokenizer.
- * <p>
- * CharFilters are chainable filters that normalize text before tokenization
- * and provide mappings between normalized text offsets and the corresponding
- * offset in the original text.
- * </p>
+ *
+ * <p>CharFilters are chainable filters that normalize text before tokenization and provide mappings
+ * between normalized text offsets and the corresponding offset in the original text.
+ *
* <H2>CharFilter offset mappings</H2>
- * <p>
- * CharFilters modify an input stream via a series of substring
- * replacements (including deletions and insertions) to produce an output
- * stream. There are three possible replacement cases: the replacement
- * string has the same length as the original substring; the replacement
- * is shorter; and the replacement is longer. In the latter two cases
- * (when the replacement has a different length than the original),
- * one or more offset correction mappings are required.
- * </p>
- * <p>
- * When the replacement is shorter than the original (e.g. when the
- * replacement is the empty string), a single offset correction mapping
- * should be added at the replacement's end offset in the output stream.
- * The <code>cumulativeDiff</code> parameter to the
- * <code>addOffCorrectMapping()</code> method will be the sum of all
- * previous replacement offset adjustments, with the addition of the
- * difference between the lengths of the original substring and the
- * replacement string (a positive value).
- * </p>
- * <p>
- * When the replacement is longer than the original (e.g. when the
- * original is the empty string), you should add as many offset
- * correction mappings as the difference between the lengths of the
- * replacement string and the original substring, starting at the
- * end offset the original substring would have had in the output stream.
- * The <code>cumulativeDiff</code> parameter to the
- * <code>addOffCorrectMapping()</code> method will be the sum of all
- * previous replacement offset adjustments, with the addition of the
- * difference between the lengths of the original substring and the
- * replacement string so far (a negative value).
- * </p>
+ *
+ * <p>CharFilters modify an input stream via a series of substring replacements (including deletions
+ * and insertions) to produce an output stream. There are three possible replacement cases: the
+ * replacement string has the same length as the original substring; the replacement is shorter; and
+ * the replacement is longer. In the latter two cases (when the replacement has a different length
+ * than the original), one or more offset correction mappings are required.
+ *
+ * <p>When the replacement is shorter than the original (e.g. when the replacement is the empty
+ * string), a single offset correction mapping should be added at the replacement's end offset in
+ * the output stream. The <code>cumulativeDiff</code> parameter to the <code>addOffCorrectMapping()
+ * </code> method will be the sum of all previous replacement offset adjustments, with the addition
+ * of the difference between the lengths of the original substring and the replacement string (a
+ * positive value).
+ *
+ * <p>When the replacement is longer than the original (e.g. when the original is the empty string),
+ * you should add as many offset correction mappings as the difference between the lengths of the
+ * replacement string and the original substring, starting at the end offset the original substring
+ * would have had in the output stream. The <code>cumulativeDiff</code> parameter to the <code>
+ * addOffCorrectMapping()</code> method will be the sum of all previous replacement offset
+ * adjustments, with the addition of the difference between the lengths of the original substring
+ * and the replacement string so far (a negative value).
*/
package org.apache.lucene.analysis.charfilter;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
index 403f43a..4eab251 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.cjk;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -29,10 +27,9 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
/**
- * An {@link Analyzer} that tokenizes text with {@link StandardTokenizer},
- * normalizes content with {@link CJKWidthFilter}, folds case with
- * {@link LowerCaseFilter}, forms bigrams of CJK with {@link CJKBigramFilter},
- * and filters stopwords with {@link StopFilter}
+ * An {@link Analyzer} that tokenizes text with {@link StandardTokenizer}, normalizes content with
+ * {@link CJKWidthFilter}, folds case with {@link LowerCaseFilter}, forms bigrams of CJK with {@link
+ * CJKBigramFilter}, and filters stopwords with {@link StopFilter}
*
* @since 3.1
*/
@@ -40,20 +37,21 @@ public final class CJKAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default CJK stopwords.
- * <p>
- * Currently it contains some common English words that are not usually
- * useful for searching and some double-byte interpunctions.
+ *
+ * <p>Currently it contains some common English words that are not usually useful for searching
+ * and some double-byte interpunctions.
*/
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
/**
* Returns an unmodifiable instance of the default stop-words set.
+ *
* @return an unmodifiable instance of the default stop-words set.
*/
- public static CharArraySet getDefaultStopSet(){
+ public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
@@ -68,20 +66,17 @@ public final class CJKAnalyzer extends StopwordAnalyzerBase {
}
}
- /**
- * Builds an analyzer which removes words in {@link #getDefaultStopSet()}.
- */
+ /** Builds an analyzer which removes words in {@link #getDefaultStopSet()}. */
public CJKAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
-
+
/**
* Builds an analyzer with the given stop words
- *
- * @param stopwords
- * a stopword set
+ *
+ * @param stopwords a stopword set
*/
- public CJKAnalyzer(CharArraySet stopwords){
+ public CJKAnalyzer(CharArraySet stopwords) {
super(stopwords);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java
index 7d79b84..abbd5d2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.cjk;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardTokenizer;
@@ -30,28 +28,24 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.ArrayUtil;
/**
- * Forms bigrams of CJK terms that are generated from StandardTokenizer
- * or ICUTokenizer.
- * <p>
- * CJK types are set by these tokenizers, but you can also use
- * {@link #CJKBigramFilter(TokenStream, int)} to explicitly control which
- * of the CJK scripts are turned into bigrams.
- * <p>
- * By default, when a CJK character has no adjacent characters to form
- * a bigram, it is output in unigram form. If you want to always output
- * both unigrams and bigrams, set the <code>outputUnigrams</code>
- * flag in {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean)}.
+ * Forms bigrams of CJK terms that are generated from StandardTokenizer or ICUTokenizer.
+ *
+ * <p>CJK types are set by these tokenizers, but you can also use {@link
+ * #CJKBigramFilter(TokenStream, int)} to explicitly control which of the CJK scripts are turned
+ * into bigrams.
+ *
+ * <p>By default, when a CJK character has no adjacent characters to form a bigram, it is output in
+ * unigram form. If you want to always output both unigrams and bigrams, set the <code>
+ * outputUnigrams</code> flag in {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean)}.
* This can be used for a combined unigram+bigram approach.
- * <p>
- * Unlike ICUTokenizer, StandardTokenizer does not split at script boundaries.
- * Korean Hangul characters are treated the same as many other scripts'
- * letters, and as a result, StandardTokenizer can produce tokens that mix
- * Hangul and non-Hangul characters, e.g. "한국abc". Such mixed-script tokens
- * are typed as <code><ALPHANUM></code> rather than
- * <code><HANGUL></code>, and as a result, will not be converted to
- * bigrams by CJKBigramFilter.
*
- * In all cases, all non-CJK input is passed thru unmodified.
+ * <p>Unlike ICUTokenizer, StandardTokenizer does not split at script boundaries. Korean Hangul
+ * characters are treated the same as many other scripts' letters, and as a result,
+ * StandardTokenizer can produce tokens that mix Hangul and non-Hangul characters, e.g. "한국abc".
+ * Such mixed-script tokens are typed as <code><ALPHANUM></code> rather than <code>
+ * <HANGUL></code>, and as a result, will not be converted to bigrams by CJKBigramFilter.
+ *
+ * <p>In all cases, all non-CJK input is passed thru unmodified.
*/
public final class CJKBigramFilter extends TokenFilter {
// configuration
@@ -70,12 +64,15 @@ public final class CJKBigramFilter extends TokenFilter {
public static final String SINGLE_TYPE = "<SINGLE>";
// the types from standardtokenizer
- private static final String HAN_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.IDEOGRAPHIC];
- private static final String HIRAGANA_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HIRAGANA];
- private static final String KATAKANA_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.KATAKANA];
+ private static final String HAN_TYPE =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.IDEOGRAPHIC];
+ private static final String HIRAGANA_TYPE =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HIRAGANA];
+ private static final String KATAKANA_TYPE =
+ StandardTokenizer.TOKEN_TYPES[StandardTokenizer.KATAKANA];
private static final String HANGUL_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HANGUL];
-
- // sentinel value for ignoring a script
+
+ // sentinel value for ignoring a script
private static final Object NO = new Object();
// these are set to either their type or NO if we want to pass them thru
@@ -83,17 +80,18 @@ public final class CJKBigramFilter extends TokenFilter {
private final Object doHiragana;
private final Object doKatakana;
private final Object doHangul;
-
+
// true if we should output unigram tokens always
private final boolean outputUnigrams;
private boolean ngramState; // false = output unigram, true = output bigram
-
+
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionIncrementAttribute posIncAtt =
+ addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLengthAtt = addAttribute(PositionLengthAttribute.class);
-
+
// buffers containing codepoint and offsets in parallel
int buffer[] = new int[8];
int startOffset[] = new int[8];
@@ -102,65 +100,66 @@ public final class CJKBigramFilter extends TokenFilter {
int bufferLen;
// current buffer index
int index;
-
+
// the last end offset, to determine if we should bigram across tokens
int lastEndOffset;
-
+
private boolean exhausted;
-
- /**
- * Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int)
- * CJKBigramFilter(in, HAN | HIRAGANA | KATAKANA | HANGUL)}
+
+ /**
+ * Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int) CJKBigramFilter(in, HAN |
+ * HIRAGANA | KATAKANA | HANGUL)}
*/
public CJKBigramFilter(TokenStream in) {
this(in, HAN | HIRAGANA | KATAKANA | HANGUL);
}
-
- /**
- * Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean)
- * CJKBigramFilter(in, flags, false)}
+
+ /**
+ * Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean) CJKBigramFilter(in,
+ * flags, false)}
*/
public CJKBigramFilter(TokenStream in, int flags) {
this(in, flags, false);
}
-
+
/**
- * Create a new CJKBigramFilter, specifying which writing systems should be bigrammed,
- * and whether or not unigrams should also be output.
- * @param flags OR'ed set from {@link CJKBigramFilter#HAN}, {@link CJKBigramFilter#HIRAGANA},
- * {@link CJKBigramFilter#KATAKANA}, {@link CJKBigramFilter#HANGUL}
+ * Create a new CJKBigramFilter, specifying which writing systems should be bigrammed, and whether
+ * or not unigrams should also be output.
+ *
+ * @param flags OR'ed set from {@link CJKBigramFilter#HAN}, {@link CJKBigramFilter#HIRAGANA},
+ * {@link CJKBigramFilter#KATAKANA}, {@link CJKBigramFilter#HANGUL}
* @param outputUnigrams true if unigrams for the selected writing systems should also be output.
- * when this is false, this is only done when there are no adjacent characters to form
- * a bigram.
+ * when this is false, this is only done when there are no adjacent characters to form a
+ * bigram.
*/
public CJKBigramFilter(TokenStream in, int flags, boolean outputUnigrams) {
super(in);
- doHan = (flags & HAN) == 0 ? NO : HAN_TYPE;
+ doHan = (flags & HAN) == 0 ? NO : HAN_TYPE;
doHiragana = (flags & HIRAGANA) == 0 ? NO : HIRAGANA_TYPE;
doKatakana = (flags & KATAKANA) == 0 ? NO : KATAKANA_TYPE;
- doHangul = (flags & HANGUL) == 0 ? NO : HANGUL_TYPE;
+ doHangul = (flags & HANGUL) == 0 ? NO : HANGUL_TYPE;
this.outputUnigrams = outputUnigrams;
}
-
+
/*
- * much of this complexity revolves around handling the special case of a
- * "lone cjk character" where cjktokenizer would output a unigram. this
+ * much of this complexity revolves around handling the special case of a
+ * "lone cjk character" where cjktokenizer would output a unigram. this
* is also the only time we ever have to captureState.
*/
@Override
public boolean incrementToken() throws IOException {
while (true) {
if (hasBufferedBigram()) {
-
+
// case 1: we have multiple remaining codepoints buffered,
// so we can emit a bigram here.
-
+
if (outputUnigrams) {
// when also outputting unigrams, we output the unigram first,
// then rewind back to revisit the bigram.
// so an input of ABC is A + (rewind)AB + B + (rewind)BC + C
- // the logic in hasBufferedUnigram ensures we output the C,
+ // the logic in hasBufferedUnigram ensures we output the C,
// even though it did actually have adjacent CJK characters.
if (ngramState) {
@@ -175,23 +174,23 @@ public final class CJKBigramFilter extends TokenFilter {
}
return true;
} else if (doNext()) {
-
+
// case 2: look at the token type. should we form any n-grams?
-
+
String type = typeAtt.type();
if (type == doHan || type == doHiragana || type == doKatakana || type == doHangul) {
-
+
// acceptable CJK type: we form n-grams from these.
// as long as the offsets are aligned, we just add these to our current buffer.
// otherwise, we clear the buffer and start over.
-
+
if (offsetAtt.startOffset() != lastEndOffset) { // unaligned, clear queue
if (hasBufferedUnigram()) {
-
+
// we have a buffered unigram, and we peeked ahead to see if we could form
- // a bigram, but we can't, because the offsets are unaligned. capture the state
+ // a bigram, but we can't, because the offsets are unaligned. capture the state
// of this peeked data to be revisited next time thru the loop, and dump our unigram.
-
+
loneState = captureState();
flushUnigram();
return true;
@@ -201,15 +200,15 @@ public final class CJKBigramFilter extends TokenFilter {
}
refill();
} else {
-
+
// not a CJK type: we just return these as-is.
-
+
if (hasBufferedUnigram()) {
-
+
// we have a buffered unigram, and we peeked ahead to see if we could form
- // a bigram, but we can't, because it's not a CJK type. capture the state
+ // a bigram, but we can't, because it's not a CJK type. capture the state
// of this peeked data to be revisited next time thru the loop, and dump our unigram.
-
+
loneState = captureState();
flushUnigram();
return true;
@@ -217,12 +216,12 @@ public final class CJKBigramFilter extends TokenFilter {
return true;
}
} else {
-
- // case 3: we have only zero or 1 codepoints buffered,
+
+ // case 3: we have only zero or 1 codepoints buffered,
// so not enough to form a bigram. But, we also have no
// more input. So if we have a buffered codepoint, emit
// a unigram, otherwise, it's end of stream.
-
+
if (hasBufferedUnigram()) {
flushUnigram(); // flush our remaining unigram
return true;
@@ -231,12 +230,10 @@ public final class CJKBigramFilter extends TokenFilter {
}
}
}
-
+
private State loneState; // rarely used: only for "lone cjk characters", where we emit unigrams
-
- /**
- * looks at next input token, returning false is none is available
- */
+
+ /** looks at next input token, returning false is none is available */
private boolean doNext() throws IOException {
if (loneState != null) {
restoreState(loneState);
@@ -253,10 +250,8 @@ public final class CJKBigramFilter extends TokenFilter {
}
}
}
-
- /**
- * refills buffers with new data from the current token.
- */
+
+ /** refills buffers with new data from the current token. */
private void refill() {
// compact buffers to keep them smallish if they become large
// just a safety check, but technically we only need the last codepoint
@@ -273,7 +268,7 @@ public final class CJKBigramFilter extends TokenFilter {
int len = termAtt.length();
int start = offsetAtt.startOffset();
int end = offsetAtt.endOffset();
-
+
int newSize = bufferLen + len;
buffer = ArrayUtil.grow(buffer, newSize);
startOffset = ArrayUtil.grow(startOffset, newSize);
@@ -300,17 +295,17 @@ public final class CJKBigramFilter extends TokenFilter {
}
}
- /**
- * Flushes a bigram token to output from our buffer
- * This is the normal case, e.g. ABC -> AB BC
+ /**
+ * Flushes a bigram token to output from our buffer This is the normal case, e.g. ABC -> AB BC
*/
private void flushBigram() {
clearAttributes();
- char termBuffer[] = termAtt.resizeBuffer(4); // maximum bigram length in code units (2 supplementaries)
+ char termBuffer[] =
+ termAtt.resizeBuffer(4); // maximum bigram length in code units (2 supplementaries)
int len1 = Character.toChars(buffer[index], termBuffer, 0);
- int len2 = len1 + Character.toChars(buffer[index+1], termBuffer, len1);
+ int len2 = len1 + Character.toChars(buffer[index + 1], termBuffer, len1);
termAtt.setLength(len2);
- offsetAtt.setOffset(startOffset[index], endOffset[index+1]);
+ offsetAtt.setOffset(startOffset[index], endOffset[index + 1]);
typeAtt.setType(DOUBLE_TYPE);
// when outputting unigrams, all bigrams are synonyms that span two unigrams
if (outputUnigrams) {
@@ -319,12 +314,11 @@ public final class CJKBigramFilter extends TokenFilter {
}
index++;
}
-
- /**
- * Flushes a unigram token to output from our buffer.
- * This happens when we encounter isolated CJK characters, either the whole
- * CJK string is a single character, or we encounter a CJK character surrounded
- * by space, punctuation, english, etc, but not beside any other CJK.
+
+ /**
+ * Flushes a unigram token to output from our buffer. This happens when we encounter isolated CJK
+ * characters, either the whole CJK string is a single character, or we encounter a CJK character
+ * surrounded by space, punctuation, english, etc, but not beside any other CJK.
*/
private void flushUnigram() {
clearAttributes();
@@ -335,18 +329,15 @@ public final class CJKBigramFilter extends TokenFilter {
typeAtt.setType(SINGLE_TYPE);
index++;
}
-
- /**
- * True if we have multiple codepoints sitting in our buffer
- */
+
+ /** True if we have multiple codepoints sitting in our buffer */
private boolean hasBufferedBigram() {
return bufferLen - index > 1;
}
/**
- * True if we have a single codepoint sitting in our buffer, where its future
- * (whether it is emitted as unigram or forms a bigram) depends upon not-yet-seen
- * inputs.
+ * True if we have a single codepoint sitting in our buffer, where its future (whether it is
+ * emitted as unigram or forms a bigram) depends upon not-yet-seen inputs.
*/
private boolean hasBufferedUnigram() {
if (outputUnigrams) {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java
index 1eceaf9..1c8699a 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java
@@ -16,25 +16,25 @@
*/
package org.apache.lucene.analysis.cjk;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
+/**
* Factory for {@link CJKBigramFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_cjk" class="solr.TextField">
* <analyzer>
* <tokenizer class="solr.StandardTokenizerFactory"/>
* <filter class="solr.CJKWidthFilterFactory"/>
* <filter class="solr.LowerCaseFilterFactory"/>
- * <filter class="solr.CJKBigramFilterFactory"
- * han="true" hiragana="true"
+ * <filter class="solr.CJKBigramFilterFactory"
+ * han="true" hiragana="true"
* katakana="true" hangul="true" outputUnigrams="false" />
* </analyzer>
* </fieldType></pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
@@ -47,7 +47,7 @@ public class CJKBigramFilterFactory extends TokenFilterFactory {
final boolean outputUnigrams;
/** Creates a new CJKBigramFilterFactory */
- public CJKBigramFilterFactory(Map<String,String> args) {
+ public CJKBigramFilterFactory(Map<String, String> args) {
super(args);
int flags = 0;
if (getBoolean(args, "han", true)) {
@@ -68,7 +68,7 @@ public class CJKBigramFilterFactory extends TokenFilterFactory {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public CJKBigramFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilter.java
index 4fb7ced..16cd6f2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilter.java
@@ -17,19 +17,19 @@
package org.apache.lucene.analysis.cjk;
-import org.apache.lucene.analysis.charfilter.BaseCharFilter;
-
import java.io.IOException;
import java.io.Reader;
+import org.apache.lucene.analysis.charfilter.BaseCharFilter;
/**
* A {@link org.apache.lucene.analysis.CharFilter} that normalizes CJK width differences:
+ *
* <ul>
* <li>Folds fullwidth ASCII variants into the equivalent basic latin
* <li>Folds halfwidth Katakana variants into the equivalent kana
* </ul>
- * <p>
- * NOTE: this char filter is the exact counterpart of {@link CJKWidthFilter}.
+ *
+ * <p>NOTE: this char filter is the exact counterpart of {@link CJKWidthFilter}.
*/
public class CJKWidthCharFilter extends BaseCharFilter {
@@ -39,30 +39,33 @@ public class CJKWidthCharFilter extends BaseCharFilter {
* as a fallback when they cannot properly combine with a preceding
* character into a composed form.
*/
- private static final char KANA_NORM[] = new char[] {
- 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5,
- 0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab,
- 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd,
- 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd,
- 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0,
- 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec,
- 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A
- };
+ private static final char KANA_NORM[] =
+ new char[] {
+ 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5,
+ 0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab,
+ 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd,
+ 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd,
+ 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0,
+ 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec,
+ 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A
+ };
/* kana combining diffs: 0x30A6-0x30FD */
- private static final byte KANA_COMBINE_VOICED[] = new byte[] {
- 78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
- 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,
- 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
- };
-
- private static final byte KANA_COMBINE_SEMI_VOICED[] = new byte[] {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2,
- 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
- };
+ private static final byte KANA_COMBINE_VOICED[] =
+ new byte[] {
+ 78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,
+ 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+ };
+
+ private static final byte KANA_COMBINE_SEMI_VOICED[] =
+ new byte[] {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2,
+ 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ };
private static final int HW_KATAKANA_VOICED_MARK = 0xFF9E;
private static final int HW_KATAKANA_SEMI_VOICED_MARK = 0xFF9F;
@@ -77,7 +80,7 @@ public class CJKWidthCharFilter extends BaseCharFilter {
@Override
public int read() throws IOException {
- while(true) {
+ while (true) {
final int ch = input.read();
if (ch == -1) {
// reached end of the input
@@ -126,9 +129,10 @@ public class CJKWidthCharFilter extends BaseCharFilter {
private int combineVoiceMark(int ch, int voiceMark) {
assert voiceMark == HW_KATAKANA_SEMI_VOICED_MARK || voiceMark == HW_KATAKANA_VOICED_MARK;
if (ch >= 0x30A6 && ch <= 0x30FD) {
- ch += (voiceMark == HW_KATAKANA_SEMI_VOICED_MARK)
- ? KANA_COMBINE_SEMI_VOICED[prevChar - 0x30A6]
- : KANA_COMBINE_VOICED[prevChar - 0x30A6];
+ ch +=
+ (voiceMark == HW_KATAKANA_SEMI_VOICED_MARK)
+ ? KANA_COMBINE_SEMI_VOICED[prevChar - 0x30A6]
+ : KANA_COMBINE_VOICED[prevChar - 0x30A6];
}
return ch;
}
@@ -136,7 +140,7 @@ public class CJKWidthCharFilter extends BaseCharFilter {
@Override
public int read(char[] cbuf, int off, int len) throws IOException {
int numRead = 0;
- for(int i = off; i < off + len; i++) {
+ for (int i = off; i < off + len; i++) {
int c = read();
if (c == -1) break;
cbuf[i] = (char) c;
@@ -144,5 +148,4 @@ public class CJKWidthCharFilter extends BaseCharFilter {
}
return numRead == 0 ? -1 : numRead;
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilterFactory.java
index 4f8bf09..bec98c9 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthCharFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.cjk;
-
-import org.apache.lucene.analysis.CharFilterFactory;
-
import java.io.Reader;
import java.util.Map;
+import org.apache.lucene.analysis.CharFilterFactory;
/**
* Factory for {@link CJKWidthCharFilter}.
+ *
* @lucene.spi {@value #NAME}
*/
public class CJKWidthCharFilterFactory extends CharFilterFactory {
@@ -32,7 +31,7 @@ public class CJKWidthCharFilterFactory extends CharFilterFactory {
public static final String NAME = "cjkWidth";
/** Creates a new CJKWidthCharFilterFactory */
- public CJKWidthCharFilterFactory(Map<String,String> args) {
+ public CJKWidthCharFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@@ -53,5 +52,4 @@ public class CJKWidthCharFilterFactory extends CharFilterFactory {
public Reader normalize(Reader input) {
return create(input);
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java
index 485ac63..9997576 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java
@@ -16,9 +16,7 @@
*/
package org.apache.lucene.analysis.cjk;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -26,33 +24,34 @@ import org.apache.lucene.analysis.util.StemmerUtil;
/**
* A {@link TokenFilter} that normalizes CJK width differences:
+ *
* <ul>
* <li>Folds fullwidth ASCII variants into the equivalent basic latin
* <li>Folds halfwidth Katakana variants into the equivalent kana
* </ul>
- * <p>
- * NOTE: this filter can be viewed as a (practical) subset of NFKC/NFKD
- * Unicode normalization. See the normalization support in the ICU package
- * for full normalization.
+ *
+ * <p>NOTE: this filter can be viewed as a (practical) subset of NFKC/NFKD Unicode normalization.
+ * See the normalization support in the ICU package for full normalization.
*/
public final class CJKWidthFilter extends TokenFilter {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-
- /* halfwidth kana mappings: 0xFF65-0xFF9D
+
+ /* halfwidth kana mappings: 0xFF65-0xFF9D
*
* note: 0xFF9C and 0xFF9D are only mapped to 0x3099 and 0x309A
- * as a fallback when they cannot properly combine with a preceding
+ * as a fallback when they cannot properly combine with a preceding
* character into a composed form.
*/
- private static final char KANA_NORM[] = new char[] {
- 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5,
- 0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab,
- 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd,
- 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd,
- 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0,
- 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec,
- 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A
- };
+ private static final char KANA_NORM[] =
+ new char[] {
+ 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5,
+ 0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab,
+ 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd,
+ 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd,
+ 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0,
+ 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec,
+ 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A
+ };
public CJKWidthFilter(TokenStream input) {
super(input);
@@ -85,28 +84,30 @@ public final class CJKWidthFilter extends TokenFilter {
}
/* kana combining diffs: 0x30A6-0x30FD */
- private static final byte KANA_COMBINE_VOICED[] = new byte[] {
- 78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
- 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,
- 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
- };
-
- private static final byte KANA_COMBINE_HALF_VOICED[] = new byte[] {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2,
- 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
- };
-
+ private static final byte KANA_COMBINE_VOICED[] =
+ new byte[] {
+ 78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0,
+ 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+ };
+
+ private static final byte KANA_COMBINE_HALF_VOICED[] =
+ new byte[] {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2,
+ 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+ };
+
/** returns true if we successfully combined the voice mark */
private static boolean combine(char text[], int pos, char ch) {
- final char prev = text[pos-1];
+ final char prev = text[pos - 1];
if (prev >= 0x30A6 && prev <= 0x30FD) {
- text[pos-1] += (ch == 0xFF9F)
- ? KANA_COMBINE_HALF_VOICED[prev - 0x30A6]
- : KANA_COMBINE_VOICED[prev - 0x30A6];
- return text[pos-1] != prev;
+ text[pos - 1] +=
+ (ch == 0xFF9F)
+ ? KANA_COMBINE_HALF_VOICED[prev - 0x30A6]
+ : KANA_COMBINE_VOICED[prev - 0x30A6];
+ return text[pos - 1] != prev;
}
return false;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
index 3e522e3..8e464fd 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.cjk;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
-/**
+/**
* Factory for {@link CJKWidthFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_cjk" class="solr.TextField">
* <analyzer>
@@ -33,6 +32,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.CJKBigramFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 3.6.0
* @lucene.spi {@value #NAME}
*/
@@ -40,15 +40,15 @@ public class CJKWidthFilterFactory extends TokenFilterFactory {
/** SPI name */
public static final String NAME = "cjkWidth";
-
+
/** Creates a new CJKWidthFilterFactory */
- public CJKWidthFilterFactory(Map<String,String> args) {
+ public CJKWidthFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public CJKWidthFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/package-info.java
index 2fd4ad4..77e859c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/package-info.java
@@ -16,21 +16,28 @@
*/
/**
- * Analyzer for Chinese, Japanese, and Korean, which indexes bigrams.
- * This analyzer generates bigram terms, which are overlapping groups of two adjacent Han, Hiragana, Katakana, or Hangul characters.
- * <p>
- * Three analyzers are provided for Chinese, each of which treats Chinese text in a different way.
+ * Analyzer for Chinese, Japanese, and Korean, which indexes bigrams. This analyzer generates bigram
+ * terms, which are overlapping groups of two adjacent Han, Hiragana, Katakana, or Hangul
+ * characters.
+ *
+ * <p>Three analyzers are provided for Chinese, each of which treats Chinese text in a different
+ * way.
+ *
* <ul>
- * <li>ChineseAnalyzer (in the analyzers/cn package): Index unigrams (individual Chinese characters) as a token.
- * <li>CJKAnalyzer (in this package): Index bigrams (overlapping groups of two adjacent Chinese characters) as tokens.
- * <li>SmartChineseAnalyzer (in the analyzers/smartcn package): Index words (attempt to segment Chinese text into words) as tokens.
+ * <li>ChineseAnalyzer (in the analyzers/cn package): Index unigrams (individual Chinese
+ * characters) as a token.
+ * <li>CJKAnalyzer (in this package): Index bigrams (overlapping groups of two adjacent Chinese
+ * characters) as tokens.
+ * <li>SmartChineseAnalyzer (in the analyzers/smartcn package): Index words (attempt to segment
+ * Chinese text into words) as tokens.
* </ul>
- *
+ *
* Example phrase： "我是中国人"
+ *
* <ol>
- * <li>ChineseAnalyzer: 我－是－中－国－人</li>
- * <li>CJKAnalyzer: 我是－是中－中国－国人</li>
- * <li>SmartChineseAnalyzer: 我－是－中国－人</li>
+ * <li>ChineseAnalyzer: 我－是－中－国－人
+ * <li>CJKAnalyzer: 我是－是中－中国－国人
+ * <li>SmartChineseAnalyzer: 我－是－中国－人
* </ol>
*/
package org.apache.lucene.analysis.cjk;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
index 4b70886..e76d8c7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
@@ -16,11 +16,9 @@
*/
package org.apache.lucene.analysis.ckb;
-
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -41,29 +39,32 @@ import org.apache.lucene.util.IOUtils;
*/
public final class SoraniAnalyzer extends StopwordAnalyzerBase {
private final CharArraySet stemExclusionSet;
-
+
/** File containing default Kurdish stopwords. */
- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
-
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
/**
* Returns an unmodifiable instance of the default stop words set.
+ *
* @return default stop words set.
*/
public static CharArraySet getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
-
+
/**
- * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class
- * accesses the static final set the first time.;
+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the
+ * static final set the first time.;
*/
private static class DefaultSetHolder {
static final CharArraySet DEFAULT_STOP_SET;
static {
try {
- DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getDecodingReader(SoraniAnalyzer.class,
- DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+ DEFAULT_STOP_SET =
+ WordlistLoader.getWordSet(
+ IOUtils.getDecodingReader(
+ SoraniAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
@@ -72,16 +73,14 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
}
}
- /**
- * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
- */
+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
public SoraniAnalyzer() {
this(DefaultSetHolder.DEFAULT_STOP_SET);
}
-
+
/**
* Builds an analyzer with the given stop words.
- *
+ *
* @param stopwords a stopword set
*/
public SoraniAnalyzer(CharArraySet stopwords) {
@@ -89,10 +88,9 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
}
/**
- * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is
- * provided this analyzer will add a {@link SetKeywordMarkerFilter} before
- * stemming.
- *
+ * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is provided
+ * this analyzer will add a {@link SetKeywordMarkerFilter} before stemming.
+ *
* @param stopwords a stopword set
* @param stemExclusionSet a set of terms not to be stemmed
*/
@@ -102,17 +100,13 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
}
/**
- * Creates a
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * which tokenizes all the text in the provided {@link Reader}.
- *
- * @return A
- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from an {@link StandardTokenizer} filtered with
- * {@link SoraniNormalizationFilter},
- * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link StopFilter}
- * , {@link SetKeywordMarkerFilter} if a stem exclusion set is
- * provided and {@link SoraniStemFilter}.
+ * Creates a {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} which tokenizes all
+ * the text in the provided {@link Reader}.
+ *
+ * @return A {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from an
+ * {@link StandardTokenizer} filtered with {@link SoraniNormalizationFilter}, {@link
+ * LowerCaseFilter}, {@link DecimalDigitFilter}, {@link StopFilter}, {@link
+ * SetKeywordMarkerFilter} if a stem exclusion set is provided and {@link SoraniStemFilter}.
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
@@ -121,8 +115,7 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
result = new LowerCaseFilter(result);
result = new DecimalDigitFilter(result);
result = new StopFilter(result, stopwords);
- if(!stemExclusionSet.isEmpty())
- result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+ if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new SoraniStemFilter(result);
return new TokenStreamComponents(source, result);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilter.java
index 6819b5b..a286d24 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilter.java
@@ -16,17 +16,12 @@
*/
package org.apache.lucene.analysis.ckb;
-
import java.io.IOException;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-/**
- * A {@link TokenFilter} that applies {@link SoraniNormalizer} to normalize the
- * orthography.
- */
+/** A {@link TokenFilter} that applies {@link SoraniNormalizer} to normalize the orthography. */
public final class SoraniNormalizationFilter extends TokenFilter {
private final SoraniNormalizer normalizer = new SoraniNormalizer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@@ -41,7 +36,7 @@ public final class SoraniNormalizationFilter extends TokenFilter {
final int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
return true;
- }
+ }
return false;
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
index 26a9a68..59409f7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ckb;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link SoraniNormalizationFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ckbnormal" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -31,6 +30,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.SoraniNormalizationFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 4.7.0
* @lucene.spi {@value #NAME}
*/
@@ -40,7 +40,7 @@ public class SoraniNormalizationFilterFactory extends TokenFilterFactory {
public static final String NAME = "soraniNormalization";
/** Creates a new SoraniNormalizationFilterFactory */
- public SoraniNormalizationFilterFactory(Map<String,String> args) {
+ public SoraniNormalizationFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizer.java
index bb9187c..228909d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizer.java
@@ -18,38 +18,40 @@ package org.apache.lucene.analysis.ckb;
import static org.apache.lucene.analysis.util.StemmerUtil.delete;
-/**
+/**
* Normalizes the Unicode representation of Sorani text.
- * <p>
- * Normalization consists of:
+ *
+ * <p>Normalization consists of:
+ *
* <ul>
* <li>Alternate forms of 'y' (0064, 0649) are converted to 06CC (FARSI YEH)
* <li>Alternate form of 'k' (0643) is converted to 06A9 (KEHEH)
* <li>Alternate forms of vowel 'e' (0647+200C, word-final 0647, 0629) are converted to 06D5 (AE)
* <li>Alternate (joining) form of 'h' (06BE) is converted to 0647
- * <li>Alternate forms of 'rr' (0692, word-initial 0631) are converted to 0695 (REH WITH SMALL V BELOW)
+ * <li>Alternate forms of 'rr' (0692, word-initial 0631) are converted to 0695 (REH WITH SMALL V
+ * BELOW)
* <li>Harakat, tatweel, and formatting characters such as directional controls are removed.
* </ul>
*/
public class SoraniNormalizer {
-
+
static final char YEH = '\u064A';
static final char DOTLESS_YEH = '\u0649';
static final char FARSI_YEH = '\u06CC';
-
+
static final char KAF = '\u0643';
static final char KEHEH = '\u06A9';
-
+
static final char HEH = '\u0647';
static final char AE = '\u06D5';
static final char ZWNJ = '\u200C';
static final char HEH_DOACHASHMEE = '\u06BE';
static final char TEH_MARBUTA = '\u0629';
-
+
static final char REH = '\u0631';
static final char RREH = '\u0695';
static final char RREH_ABOVE = '\u0692';
-
+
static final char TATWEEL = '\u0640';
static final char FATHATAN = '\u064B';
static final char DAMMATAN = '\u064C';
@@ -62,7 +64,7 @@ public class SoraniNormalizer {
/**
* Normalize an input buffer of Sorani text
- *
+ *
* @param s input buffer
* @param len length of input buffer
* @return length of input buffer after normalization
@@ -78,14 +80,14 @@ public class SoraniNormalizer {
s[i] = KEHEH;
break;
case ZWNJ:
- if (i > 0 && s[i-1] == HEH) {
- s[i-1] = AE;
+ if (i > 0 && s[i - 1] == HEH) {
+ s[i - 1] = AE;
}
len = delete(s, i, len);
i--;
break;
case HEH:
- if (i == len-1) {
+ if (i == len - 1) {
s[i] = AE;
}
break;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilter.java
index 0d7a207..962cf2d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilter.java
@@ -16,29 +16,27 @@
*/
package org.apache.lucene.analysis.ckb;
-
import java.io.IOException;
-
-import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // javadoc @link
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // javadoc @link
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
/**
* A {@link TokenFilter} that applies {@link SoraniStemmer} to stem Sorani words.
- * <p>
- * To prevent terms from being stemmed use an instance of
- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
- * the {@link KeywordAttribute} before this {@link TokenStream}.
- * </p>
- * @see SetKeywordMarkerFilter */
-
+ *
+ * <p>To prevent terms from being stemmed use an instance of {@link SetKeywordMarkerFilter} or a
+ * custom {@link TokenFilter} that sets the {@link KeywordAttribute} before this {@link
+ * TokenStream}.
+ *
+ * @see SetKeywordMarkerFilter
+ */
public final class SoraniStemFilter extends TokenFilter {
private final SoraniStemmer stemmer = new SoraniStemmer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
-
+
public SoraniStemFilter(TokenStream input) {
super(input);
}
@@ -46,7 +44,7 @@ public final class SoraniStemFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- if(!keywordAttr.isKeyword()) {
+ if (!keywordAttr.isKeyword()) {
final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilterFactory.java
index ca12bc9..00ce270 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemFilterFactory.java
@@ -16,14 +16,13 @@
*/
package org.apache.lucene.analysis.ckb;
-
import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link SoraniStemFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_ckbstem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -32,6 +31,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.SoraniStemFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 4.7.0
* @lucene.spi {@value #NAME}
*/
@@ -41,7 +41,7 @@ public class SoraniStemFilterFactory extends TokenFilterFactory {
public static final String NAME = "soraniStem";
/** Creates a new SoraniStemFilterFactory */
- public SoraniStemFilterFactory(Map<String,String> args) {
+ public SoraniStemFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemmer.java
index 70c5e7f..b7a8df5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniStemmer.java
@@ -16,17 +16,14 @@
*/
package org.apache.lucene.analysis.ckb;
-
import static org.apache.lucene.analysis.util.StemmerUtil.endsWith;
-/**
- * Light stemmer for Sorani
- */
+/** Light stemmer for Sorani */
public class SoraniStemmer {
-
+
/**
* Stem an input buffer of Sorani text.
- *
+ *
* @param s input buffer
* @param len length of input buffer
* @return length of input buffer after normalization
@@ -40,63 +37,64 @@ public class SoraniStemmer {
} else if (len > 6 && endsWith(s, len, "ەوە")) {
len -= 3;
}
-
+
// possessive pronoun
- if (len > 6 && (endsWith(s, len, "مان") || endsWith(s, len, "یان") || endsWith(s, len, "تان"))) {
+ if (len > 6
+ && (endsWith(s, len, "مان") || endsWith(s, len, "یان") || endsWith(s, len, "تان"))) {
len -= 3;
}
-
+
// indefinite singular ezafe
if (len > 6 && endsWith(s, len, "ێکی")) {
- return len-3;
+ return len - 3;
} else if (len > 7 && endsWith(s, len, "یەکی")) {
- return len-4;
+ return len - 4;
}
// indefinite singular
if (len > 5 && endsWith(s, len, "ێک")) {
- return len-2;
+ return len - 2;
} else if (len > 6 && endsWith(s, len, "یەک")) {
- return len-3;
+ return len - 3;
}
// definite singular
else if (len > 6 && endsWith(s, len, "ەکە")) {
- return len-3;
+ return len - 3;
} else if (len > 5 && endsWith(s, len, "کە")) {
- return len-2;
+ return len - 2;
}
// definite plural
else if (len > 7 && endsWith(s, len, "ەکان")) {
- return len-4;
+ return len - 4;
} else if (len > 6 && endsWith(s, len, "کان")) {
- return len-3;
+ return len - 3;
}
// indefinite plural ezafe
else if (len > 7 && endsWith(s, len, "یانی")) {
- return len-4;
+ return len - 4;
} else if (len > 6 && endsWith(s, len, "انی")) {
- return len-3;
+ return len - 3;
}
// indefinite plural
else if (len > 6 && endsWith(s, len, "یان")) {
- return len-3;
+ return len - 3;
} else if (len > 5 && endsWith(s, len, "ان")) {
- return len-2;
- }
+ return len - 2;
+ }
// demonstrative plural
else if (len > 7 && endsWith(s, len, "یانە")) {
- return len-4;
+ return len - 4;
} else if (len > 6 && endsWith(s, len, "انە")) {
- return len-3;
+ return len - 3;
}
// demonstrative singular
else if (len > 5 && (endsWith(s, len, "ایە") || endsWith(s, len, "ەیە"))) {
- return len-2;
+ return len - 2;
} else if (len > 4 && endsWith(s, len, "ە")) {
- return len-1;
+ return len - 1;
}
// absolute singular ezafe
else if (len > 4 && endsWith(s, len, "ی")) {
- return len-1;
+ return len - 1;
}
return len;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/package-info.java
index 5fccddf..97878e3 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Analyzer for Sorani Kurdish.
- */
+/** Analyzer for Sorani Kurdish. */
package org.apache.lucene.analysis.ckb;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicAnalyzer.java
index f646e1e..e76803cc 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicAnalyzer.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.classic;
-
import java.io.IOException;
import java.io.Reader;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
@@ -30,13 +28,11 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
/**
- * Filters {@link ClassicTokenizer} with {@link ClassicFilter}, {@link
- * LowerCaseFilter} and {@link StopFilter}, using a list of
- * English stop words.
- *
- * ClassicAnalyzer was named StandardAnalyzer in Lucene versions prior to 3.1.
- * As of 3.1, {@link StandardAnalyzer} implements Unicode text segmentation,
- * as specified by UAX#29.
+ * Filters {@link ClassicTokenizer} with {@link ClassicFilter}, {@link LowerCaseFilter} and {@link
+ * StopFilter}, using a list of English stop words.
+ *
+ * <p>ClassicAnalyzer was named StandardAnalyzer in Lucene versions prior to 3.1. As of 3.1, {@link
+ * StandardAnalyzer} implements Unicode text segmentation, as specified by UAX#29.
*
* @since 3.1
*/
@@ -47,43 +43,45 @@ public final class ClassicAnalyzer extends StopwordAnalyzerBase {
private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
- /** An unmodifiable set containing some common English words that are usually not
- useful for searching. */
+ /**
+ * An unmodifiable set containing some common English words that are usually not useful for
+ * searching.
+ */
public static final CharArraySet STOP_WORDS_SET = EnglishAnalyzer.ENGLISH_STOP_WORDS_SET;
- /** Builds an analyzer with the given stop words.
- * @param stopWords stop words */
+ /**
+ * Builds an analyzer with the given stop words.
+ *
+ * @param stopWords stop words
+ */
public ClassicAnalyzer(CharArraySet stopWords) {
super(stopWords);
}
- /** Builds an analyzer with the default stop words ({@link
- * #STOP_WORDS_SET}).
- */
+ /** Builds an analyzer with the default stop words ({@link #STOP_WORDS_SET}). */
public ClassicAnalyzer() {
this(STOP_WORDS_SET);
}
- /** Builds an analyzer with the stop words from the given reader.
+ /**
+ * Builds an analyzer with the stop words from the given reader.
+ *
* @see WordlistLoader#getWordSet(Reader)
- * @param stopwords Reader to read stop words from */
+ * @param stopwords Reader to read stop words from
+ */
public ClassicAnalyzer(Reader stopwords) throws IOException {
this(loadStopwordSet(stopwords));
}
/**
- * Set maximum allowed token length. If a token is seen
- * that exceeds this length then it is discarded. This
- * setting only takes effect the next time tokenStream or
- * tokenStream is called.
+ * Set maximum allowed token length. If a token is seen that exceeds this length then it is
+ * discarded. This setting only takes effect the next time tokenStream or tokenStream is called.
*/
public void setMaxTokenLength(int length) {
maxTokenLength = length;
}
-
- /**
- * @see #setMaxTokenLength
- */
+
+ /** @see #setMaxTokenLength */
public int getMaxTokenLength() {
return maxTokenLength;
}
@@ -95,10 +93,12 @@ public final class ClassicAnalyzer extends StopwordAnalyzerBase {
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
- return new TokenStreamComponents(r -> {
- src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- src.setReader(r);
- }, tok);
+ return new TokenStreamComponents(
+ r -> {
+ src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
+ src.setReader(r);
+ },
+ tok);
}
@Override
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilter.java
index 9de1235..213e052 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilter.java
@@ -16,14 +16,12 @@
*/
package org.apache.lucene.analysis.classic;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
/** Normalizes tokens extracted with {@link ClassicTokenizer}. */
-
public class ClassicFilter extends TokenFilter {
/** Construct filtering <i>in</i>. */
@@ -31,15 +29,19 @@ public class ClassicFilter extends TokenFilter {
super(in);
}
- private static final String APOSTROPHE_TYPE = ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.APOSTROPHE];
+ private static final String APOSTROPHE_TYPE =
+ ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.APOSTROPHE];
private static final String ACRONYM_TYPE = ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.ACRONYM];
// this filters uses attribute type
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-
- /** Returns the next token in the stream, or null at EOS.
+
+ /**
+ * Returns the next token in the stream, or null at EOS.
+ *
* <p>Removes <code>'s</code> from the end of words.
+ *
* <p>Removes dots from acronyms.
*/
@Override
@@ -52,18 +54,18 @@ public class ClassicFilter extends TokenFilter {
final int bufferLength = termAtt.length();
final String type = typeAtt.type();
- if (type == APOSTROPHE_TYPE && // remove 's
- bufferLength >= 2 &&
- buffer[bufferLength-2] == '\'' &&
- (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
+ // remove 's
+ if (type == APOSTROPHE_TYPE
+ && bufferLength >= 2
+ && buffer[bufferLength - 2] == '\''
+ && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S')) {
// Strip last 2 characters off
termAtt.setLength(bufferLength - 2);
- } else if (type == ACRONYM_TYPE) { // remove dots
+ } else if (type == ACRONYM_TYPE) { // remove dots
int upto = 0;
- for(int i=0;i<bufferLength;i++) {
+ for (int i = 0; i < bufferLength; i++) {
char c = buffer[i];
- if (c != '.')
- buffer[upto++] = c;
+ if (c != '.') buffer[upto++] = c;
}
termAtt.setLength(upto);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilterFactory.java
index 75d4865..4c153bf 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicFilterFactory.java
@@ -16,15 +16,14 @@
*/
package org.apache.lucene.analysis.classic;
-
import java.util.Map;
-
import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
/**
* Factory for {@link ClassicFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_clssc" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -32,6 +31,7 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* <filter class="solr.ClassicFilterFactory"/>
* </analyzer>
* </fieldType></pre>
+ *
* @since 3.1.0
* @lucene.spi {@value #NAME}
*/
@@ -39,15 +39,15 @@ public class ClassicFilterFactory extends TokenFilterFactory {
/** SPI name */
public static final String NAME = "classic";
-
+
/** Creates a new ClassicFilterFactory */
- public ClassicFilterFactory(Map<String,String> args) {
+ public ClassicFilterFactory(Map<String, String> args) {
super(args);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public ClassicFilterFactory() {
throw defaultCtorException();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizer.java
index 261d843..50f5e43 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizer.java
@@ -18,71 +18,69 @@
package org.apache.lucene.analysis.classic;
import java.io.IOException;
-
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeFactory;
-/** A grammar-based tokenizer constructed with JFlex
+/**
+ * A grammar-based tokenizer constructed with JFlex
*
- * <p> This should be a good tokenizer for most European-language documents:
+ * <p>This should be a good tokenizer for most European-language documents:
*
* <ul>
- * <li>Splits words at punctuation characters, removing punctuation. However, a
- * dot that's not followed by whitespace is considered part of a token.
- * <li>Splits words at hyphens, unless there's a number in the token, in which case
- * the whole token is interpreted as a product number and is not split.
+ * <li>Splits words at punctuation characters, removing punctuation. However, a dot that's not
+ * followed by whitespace is considered part of a token.
+ * <li>Splits words at hyphens, unless there's a number in the token, in which case the whole
+ * token is interpreted as a product number and is not split.
* <li>Recognizes email addresses and internet hostnames as one token.
* </ul>
*
- * <p>Many applications have specific tokenizer needs. If this tokenizer does
- * not suit your application, please consider copying this source code
- * directory to your project and maintaining your own grammar-based tokenizer.
+ * <p>Many applications have specific tokenizer needs. If this tokenizer does not suit your
+ * application, please consider copying this source code directory to your project and maintaining
+ * your own grammar-based tokenizer.
*
- * ClassicTokenizer was named StandardTokenizer in Lucene versions prior to 3.1.
- * As of 3.1, {@link StandardTokenizer} implements Unicode text segmentation,
- * as specified by UAX#29.
+ * <p>ClassicTokenizer was named StandardTokenizer in Lucene versions prior to 3.1. As of 3.1,
+ * {@link StandardTokenizer} implements Unicode text segmentation, as specified by UAX#29.
*/
-
public final class ClassicTokenizer extends Tokenizer {
/** A private instance of the JFlex-constructed scanner */
private ClassicTokenizerImpl scanner;
- public static final int ALPHANUM = 0;
- public static final int APOSTROPHE = 1;
- public static final int ACRONYM = 2;
- public static final int COMPANY = 3;
- public static final int EMAIL = 4;
- public static final int HOST = 5;
- public static final int NUM = 6;
- public static final int CJ = 7;
+ public static final int ALPHANUM = 0;
+ public static final int APOSTROPHE = 1;
+ public static final int ACRONYM = 2;
+ public static final int COMPANY = 3;
+ public static final int EMAIL = 4;
+ public static final int HOST = 5;
+ public static final int NUM = 6;
+ public static final int CJ = 7;
- public static final int ACRONYM_DEP = 8;
+ public static final int ACRONYM_DEP = 8;
/** String token types that correspond to token type int constants */
- public static final String [] TOKEN_TYPES = new String [] {
- "<ALPHANUM>",
- "<APOSTROPHE>",
- "<ACRONYM>",
- "<COMPANY>",
- "<EMAIL>",
- "<HOST>",
- "<NUM>",
- "<CJ>",
- "<ACRONYM_DEP>"
- };
-
+ public static final String[] TOKEN_TYPES =
+ new String[] {
+ "<ALPHANUM>",
+ "<APOSTROPHE>",
+ "<ACRONYM>",
+ "<COMPANY>",
+ "<EMAIL>",
+ "<HOST>",
+ "<NUM>",
+ "<CJ>",
+ "<ACRONYM_DEP>"
+ };
+
private int skippedPositions;
private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
- /** Set the max allowed token length. Any token longer
- * than this is skipped. */
+ /** Set the max allowed token length. Any token longer than this is skipped. */
public void setMaxTokenLength(int length) {
if (length < 1) {
throw new IllegalArgumentException("maxTokenLength must be greater than zero");
@@ -96,18 +94,16 @@ public final class ClassicTokenizer extends Tokenizer {
}
/**
- * Creates a new instance of the {@link ClassicTokenizer}. Attaches
- * the <code>input</code> to the newly created JFlex scanner.
+ * Creates a new instance of the {@link ClassicTokenizer}. Attaches the <code>input</code> to the
+ * newly created JFlex scanner.
*
- * See http://issues.apache.org/jira/browse/LUCENE-1068
+ * <p>See http://issues.apache.org/jira/browse/LUCENE-1068
*/
public ClassicTokenizer() {
init();
}
- /**
- * Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}
- */
+ /** Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeFactory} */
public ClassicTokenizer(AttributeFactory factory) {
super(factory);
init();
@@ -121,7 +117,8 @@ public final class ClassicTokenizer extends Tokenizer {
// term offset, positionIncrement and type
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt =
+ addAttribute(PositionIncrementAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
/*
@@ -134,7 +131,7 @@ public final class ClassicTokenizer extends Tokenizer {
clearAttributes();
skippedPositions = 0;
- while(true) {
+ while (true) {
int tokenType = scanner.getNextToken();
if (tokenType == ClassicTokenizerImpl.YYEOF) {
@@ -142,10 +139,10 @@ public final class ClassicTokenizer extends Tokenizer {
}
if (scanner.yylength() <= maxTokenLength) {
- posIncrAtt.setPositionIncrement(skippedPositions+1);
+ posIncrAtt.setPositionIncrement(skippedPositions + 1);
scanner.getText(termAtt);
final int start = scanner.yychar();
- offsetAtt.setOffset(correctOffset(start), correctOffset(start+termAtt.length()));
+ offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
if (tokenType == ClassicTokenizer.ACRONYM_DEP) {
typeAtt.setType(ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.HOST]);
@@ -160,7 +157,7 @@ public final class ClassicTokenizer extends Tokenizer {
skippedPositions++;
}
}
-
+
@Override
public final void end() throws IOException {
super.end();
@@ -168,9 +165,9 @@ public final class ClassicTokenizer extends Tokenizer {
int finalOffset = correctOffset(scanner.yychar() + scanner.yylength());
offsetAtt.setOffset(finalOffset, finalOffset);
// adjust any skipped tokens
- posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+skippedPositions);
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
}
-
+
@Override
public void close() throws IOException {
super.close();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerFactory.java
index 42ded7c..d62dbf9 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerFactory.java
@@ -16,15 +16,14 @@
*/
package org.apache.lucene.analysis.classic;
-
+import java.util.Map;
import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.AttributeFactory;
-import java.util.Map;
-
/**
* Factory for {@link ClassicTokenizer}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_clssc" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -43,7 +42,7 @@ public class ClassicTokenizerFactory extends TokenizerFactory {
private final int maxTokenLength;
/** Creates a new ClassicTokenizerFactory */
- public ClassicTokenizerFactory(Map<String,String> args) {
+ public ClassicTokenizerFactory(Map<String, String> args) {
super(args);
maxTokenLength = getInt(args, "maxTokenLength", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
if (!args.isEmpty()) {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java
index 163027b..9f526a9 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java
@@ -21,11 +21,8 @@ package org.apache.lucene.analysis.classic;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-/**
- * This class implements the classic lucene StandardTokenizer up until 3.0
- */
+/** This class implements the classic lucene StandardTokenizer up until 3.0 */
@SuppressWarnings("fallthrough")
-
class ClassicTokenizerImpl {
/** This character denotes the end of file */
@@ -38,136 +35,125 @@ class ClassicTokenizerImpl {
public static final int YYINITIAL = 0;
/**
- * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
- * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
- * at the beginning of a line
- * l is of the form l = 2*k, k a non negative integer
+ * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l ZZ_LEXSTATE[l+1] is the state in
+ * the DFA for the lexical state l at the beginning of a line l is of the form l = 2*k, k a non
+ * negative integer
*/
- private static final int ZZ_LEXSTATE[] = {
- 0, 0
- };
-
- /**
- * Translates characters to character classes
- */
- private static final String ZZ_CMAP_PACKED =
- "\46\0\1\5\1\3\4\0\1\11\1\7\1\4\1\11\12\2\6\0"+
- "\1\6\32\12\4\0\1\10\1\0\32\12\57\0\1\12\12\0\1\12"+
- "\4\0\1\12\5\0\27\12\1\0\37\12\1\0\u0128\12\2\0\22\12"+
- "\34\0\136\12\2\0\11\12\2\0\7\12\16\0\2\12\16\0\5\12"+
- "\11\0\1\12\213\0\1\12\13\0\1\12\1\0\3\12\1\0\1\12"+
- "\1\0\24\12\1\0\54\12\1\0\10\12\2\0\32\12\14\0\202\12"+
- "\12\0\71\12\2\0\2\12\2\0\2\12\3\0\46\12\2\0\2\12"+
- "\67\0\46\12\2\0\1\12\7\0\47\12\110\0\33\12\5\0\3\12"+
- "\56\0\32\12\5\0\13\12\25\0\12\2\7\0\143\12\1\0\1\12"+
- "\17\0\2\12\11\0\12\2\3\12\23\0\1\12\1\0\33\12\123\0"+
- "\46\12\u015f\0\65\12\3\0\1\12\22\0\1\12\7\0\12\12\4\0"+
- "\12\2\25\0\10\12\2\0\2\12\2\0\26\12\1\0\7\12\1\0"+
- "\1\12\3\0\4\12\42\0\2\12\1\0\3\12\4\0\12\2\2\12"+
- "\23\0\6\12\4\0\2\12\2\0\26\12\1\0\7\12\1\0\2\12"+
- "\1\0\2\12\1\0\2\12\37\0\4\12\1\0\1\12\7\0\12\2"+
- "\2\0\3\12\20\0\7\12\1\0\1\12\1\0\3\12\1\0\26\12"+
- "\1\0\7\12\1\0\2\12\1\0\5\12\3\0\1\12\22\0\1\12"+
- "\17\0\1\12\5\0\12\2\25\0\10\12\2\0\2\12\2\0\26\12"+
- "\1\0\7\12\1\0\2\12\2\0\4\12\3\0\1\12\36\0\2\12"+
- "\1\0\3\12\4\0\12\2\25\0\6\12\3\0\3\12\1\0\4\12"+
- "\3\0\2\12\1\0\1\12\1\0\2\12\3\0\2\12\3\0\3\12"+
- "\3\0\10\12\1\0\3\12\55\0\11\2\25\0\10\12\1\0\3\12"+
- "\1\0\27\12\1\0\12\12\1\0\5\12\46\0\2\12\4\0\12\2"+
- "\25\0\10\12\1\0\3\12\1\0\27\12\1\0\12\12\1\0\5\12"+
- "\44\0\1\12\1\0\2\12\4\0\12\2\25\0\10\12\1\0\3\12"+
- "\1\0\27\12\1\0\20\12\46\0\2\12\4\0\12\2\25\0\22\12"+
- "\3\0\30\12\1\0\11\12\1\0\1\12\2\0\7\12\71\0\1\1"+
- "\60\12\1\1\2\12\14\1\7\12\11\1\12\2\47\0\2\12\1\0"+
- "\1\12\2\0\2\12\1\0\1\12\2\0\1\12\6\0\4\12\1\0"+
- "\7\12\1\0\3\12\1\0\1\12\1\0\1\12\2\0\2\12\1\0"+
- "\4\12\1\0\2\12\11\0\1\12\2\0\5\12\1\0\1\12\11\0"+
- "\12\2\2\0\2\12\42\0\1\12\37\0\12\2\26\0\10\12\1\0"+
- "\42\12\35\0\4\12\164\0\42\12\1\0\5\12\1\0\2\12\25\0"+
- "\12\2\6\0\6\12\112\0\46\12\12\0\47\12\11\0\132\12\5\0"+
- "\104\12\5\0\122\12\6\0\7\12\1\0\77\12\1\0\1\12\1\0"+
- "\4\12\2\0\7\12\1\0\1\12\1\0\4\12\2\0\47\12\1\0"+
- "\1\12\1\0\4\12\2\0\37\12\1\0\1\12\1\0\4\12\2\0"+
- "\7\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0\7\12\1\0"+
- "\27\12\1\0\37\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0"+
- "\47\12\1\0\23\12\16\0\11\2\56\0\125\12\14\0\u026c\12\2\0"+
- "\10\12\12\0\32\12\5\0\113\12\225\0\64\12\54\0\12\2\46\0"+
- "\12\2\6\0\130\12\10\0\51\12\u0557\0\234\12\4\0\132\12\6\0"+
- "\26\12\2\0\6\12\2\0\46\12\2\0\6\12\2\0\10\12\1\0"+
- "\1\12\1\0\1\12\1\0\1\12\1\0\37\12\2\0\65\12\1\0"+
- "\7\12\1\0\1\12\3\0\3\12\1\0\7\12\3\0\4\12\2\0"+
- "\6\12\4\0\15\12\5\0\3\12\1\0\7\12\202\0\1\12\202\0"+
- "\1\12\4\0\1\12\2\0\12\12\1\0\1\12\3\0\5\12\6\0"+
- "\1\12\1\0\1\12\1\0\1\12\1\0\4\12\1\0\3\12\1\0"+
- "\7\12\u0ecb\0\2\12\52\0\5\12\12\0\1\13\124\13\10\13\2\13"+
- "\2\13\132\13\1\13\3\13\6\13\50\13\3\13\1\0\136\12\21\0"+
- "\30\12\70\0\20\13\u0100\0\200\13\200\0\u19b6\13\12\13\100\0\u51a6\13"+
- "\132\13\u048d\12\u0773\0\u2ba4\12\u215c\0\u012e\13\322\13\7\12\14\0\5\12"+
- "\5\0\1\12\1\0\12\12\1\0\15\12\1\0\5\12\1\0\1\12"+
- "\1\0\2\12\1\0\2\12\1\0\154\12\41\0\u016b\12\22\0\100\12"+
- "\2\0\66\12\50\0\14\12\164\0\3\12\1\0\1\12\1\0\207\12"+
- "\23\0\12\2\7\0\32\12\6\0\32\12\12\0\1\13\72\13\37\12"+
- "\3\0\6\12\2\0\6\12\2\0\6\12\2\0\3\12\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\63\0";
-
- /**
- * Translates characters to character classes
- */
- private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
-
- /**
- * Translates DFA states to action switch labels.
- */
- private static final int [] ZZ_ACTION = zzUnpackAction();
+ private static final int ZZ_LEXSTATE[] = {0, 0};
+
+ /** Translates characters to character classes */
+ private static final String ZZ_CMAP_PACKED =
+ "\46\0\1\5\1\3\4\0\1\11\1\7\1\4\1\11\12\2\6\0"
+ + "\1\6\32\12\4\0\1\10\1\0\32\12\57\0\1\12\12\0\1\12"
+ + "\4\0\1\12\5\0\27\12\1\0\37\12\1\0\u0128\12\2\0\22\12"
+ + "\34\0\136\12\2\0\11\12\2\0\7\12\16\0\2\12\16\0\5\12"
+ + "\11\0\1\12\213\0\1\12\13\0\1\12\1\0\3\12\1\0\1\12"
+ + "\1\0\24\12\1\0\54\12\1\0\10\12\2\0\32\12\14\0\202\12"
+ + "\12\0\71\12\2\0\2\12\2\0\2\12\3\0\46\12\2\0\2\12"
+ + "\67\0\46\12\2\0\1\12\7\0\47\12\110\0\33\12\5\0\3\12"
+ + "\56\0\32\12\5\0\13\12\25\0\12\2\7\0\143\12\1\0\1\12"
+ + "\17\0\2\12\11\0\12\2\3\12\23\0\1\12\1\0\33\12\123\0"
+ + "\46\12\u015f\0\65\12\3\0\1\12\22\0\1\12\7\0\12\12\4\0"
+ + "\12\2\25\0\10\12\2\0\2\12\2\0\26\12\1\0\7\12\1\0"
+ + "\1\12\3\0\4\12\42\0\2\12\1\0\3\12\4\0\12\2\2\12"
+ + "\23\0\6\12\4\0\2\12\2\0\26\12\1\0\7\12\1\0\2\12"
+ + "\1\0\2\12\1\0\2\12\37\0\4\12\1\0\1\12\7\0\12\2"
+ + "\2\0\3\12\20\0\7\12\1\0\1\12\1\0\3\12\1\0\26\12"
+ + "\1\0\7\12\1\0\2\12\1\0\5\12\3\0\1\12\22\0\1\12"
+ + "\17\0\1\12\5\0\12\2\25\0\10\12\2\0\2\12\2\0\26\12"
+ + "\1\0\7\12\1\0\2\12\2\0\4\12\3\0\1\12\36\0\2\12"
+ + "\1\0\3\12\4\0\12\2\25\0\6\12\3\0\3\12\1\0\4\12"
+ + "\3\0\2\12\1\0\1\12\1\0\2\12\3\0\2\12\3\0\3\12"
+ + "\3\0\10\12\1\0\3\12\55\0\11\2\25\0\10\12\1\0\3\12"
+ + "\1\0\27\12\1\0\12\12\1\0\5\12\46\0\2\12\4\0\12\2"
+ + "\25\0\10\12\1\0\3\12\1\0\27\12\1\0\12\12\1\0\5\12"
+ + "\44\0\1\12\1\0\2\12\4\0\12\2\25\0\10\12\1\0\3\12"
+ + "\1\0\27\12\1\0\20\12\46\0\2\12\4\0\12\2\25\0\22\12"
+ + "\3\0\30\12\1\0\11\12\1\0\1\12\2\0\7\12\71\0\1\1"
+ + "\60\12\1\1\2\12\14\1\7\12\11\1\12\2\47\0\2\12\1\0"
+ + "\1\12\2\0\2\12\1\0\1\12\2\0\1\12\6\0\4\12\1\0"
+ + "\7\12\1\0\3\12\1\0\1\12\1\0\1\12\2\0\2\12\1\0"
+ + "\4\12\1\0\2\12\11\0\1\12\2\0\5\12\1\0\1\12\11\0"
+ + "\12\2\2\0\2\12\42\0\1\12\37\0\12\2\26\0\10\12\1\0"
+ + "\42\12\35\0\4\12\164\0\42\12\1\0\5\12\1\0\2\12\25\0"
+ + "\12\2\6\0\6\12\112\0\46\12\12\0\47\12\11\0\132\12\5\0"
+ + "\104\12\5\0\122\12\6\0\7\12\1\0\77\12\1\0\1\12\1\0"
+ + "\4\12\2\0\7\12\1\0\1\12\1\0\4\12\2\0\47\12\1\0"
+ + "\1\12\1\0\4\12\2\0\37\12\1\0\1\12\1\0\4\12\2\0"
+ + "\7\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0\7\12\1\0"
+ + "\27\12\1\0\37\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0"
+ + "\47\12\1\0\23\12\16\0\11\2\56\0\125\12\14\0\u026c\12\2\0"
+ + "\10\12\12\0\32\12\5\0\113\12\225\0\64\12\54\0\12\2\46\0"
+ + "\12\2\6\0\130\12\10\0\51\12\u0557\0\234\12\4\0\132\12\6\0"
+ + "\26\12\2\0\6\12\2\0\46\12\2\0\6\12\2\0\10\12\1\0"
+ + "\1\12\1\0\1\12\1\0\1\12\1\0\37\12\2\0\65\12\1\0"
+ + "\7\12\1\0\1\12\3\0\3\12\1\0\7\12\3\0\4\12\2\0"
+ + "\6\12\4\0\15\12\5\0\3\12\1\0\7\12\202\0\1\12\202\0"
+ + "\1\12\4\0\1\12\2\0\12\12\1\0\1\12\3\0\5\12\6\0"
+ + "\1\12\1\0\1\12\1\0\1\12\1\0\4\12\1\0\3\12\1\0"
+ + "\7\12\u0ecb\0\2\12\52\0\5\12\12\0\1\13\124\13\10\13\2\13"
+ + "\2\13\132\13\1\13\3\13\6\13\50\13\3\13\1\0\136\12\21\0"
+ + "\30\12\70\0\20\13\u0100\0\200\13\200\0\u19b6\13\12\13\100\0\u51a6\13"
+ + "\132\13\u048d\12\u0773\0\u2ba4\12\u215c\0\u012e\13\322\13\7\12\14\0\5\12"
+ + "\5\0\1\12\1\0\12\12\1\0\15\12\1\0\5\12\1\0\1\12"
+ + "\1\0\2\12\1\0\2\12\1\0\154\12\41\0\u016b\12\22\0\100\12"
+ + "\2\0\66\12\50\0\14\12\164\0\3\12\1\0\1\12\1\0\207\12"
+ + "\23\0\12\2\7\0\32\12\6\0\32\12\12\0\1\13\72\13\37\12"
+ + "\3\0\6\12\2\0\6\12\2\0\6\12\2\0\3\12\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\63\0";
+
+ /** Translates characters to character classes */
+ private static final char[] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
+
+ /** Translates DFA states to action switch labels. */
+ private static final int[] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
- "\1\0\1\1\3\2\1\3\13\0\1\2\3\4\2\0"+
- "\1\5\1\0\1\5\3\4\6\5\1\6\1\4\2\7"+
- "\1\10\1\0\1\10\3\0\2\10\1\11\1\12\1\4";
+ "\1\0\1\1\3\2\1\3\13\0\1\2\3\4\2\0"
+ + "\1\5\1\0\1\5\3\4\6\5\1\6\1\4\2\7"
+ + "\1\10\1\0\1\10\3\0\2\10\1\11\1\12\1\4";
- private static int [] zzUnpackAction() {
- int [] result = new int[50];
+ private static int[] zzUnpackAction() {
+ int[] result = new int[50];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
}
- private static int zzUnpackAction(String packed, int offset, int [] result) {
- int i = 0; /* index in packed string */
- int j = offset; /* index in unpacked array */
+ private static int zzUnpackAction(String packed, int offset, int[] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
- do result[j++] = value; while (--count > 0);
+ do result[j++] = value;
+ while (--count > 0);
}
return j;
}
-
- /**
- * Translates a state to a row index in the transition table
- */
- private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
+ /** Translates a state to a row index in the transition table */
+ private static final int[] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
- "\0\0\0\14\0\30\0\44\0\60\0\14\0\74\0\110"+
- "\0\124\0\140\0\154\0\170\0\204\0\220\0\234\0\250"+
- "\0\264\0\300\0\314\0\330\0\344\0\360\0\374\0\u0108"+
- "\0\u0114\0\u0120\0\u012c\0\u0138\0\u0144\0\u0150\0\u015c\0\u0168"+
- "\0\u0174\0\u0180\0\u018c\0\u0198\0\u01a4\0\250\0\u01b0\0\u01bc"+
- "\0\u01c8\0\u01d4\0\u01e0\0\u01ec\0\u01f8\0\74\0\154\0\u0204"+
- "\0\u0210\0\u021c";
-
- private static int [] zzUnpackRowMap() {
- int [] result = new int[50];
+ "\0\0\0\14\0\30\0\44\0\60\0\14\0\74\0\110"
+ + "\0\124\0\140\0\154\0\170\0\204\0\220\0\234\0\250"
+ + "\0\264\0\300\0\314\0\330\0\344\0\360\0\374\0\u0108"
+ + "\0\u0114\0\u0120\0\u012c\0\u0138\0\u0144\0\u0150\0\u015c\0\u0168"
+ + "\0\u0174\0\u0180\0\u018c\0\u0198\0\u01a4\0\250\0\u01b0\0\u01bc"
+ + "\0\u01c8\0\u01d4\0\u01e0\0\u01ec\0\u01f8\0\74\0\154\0\u0204"
+ + "\0\u0210\0\u021c";
+
+ private static int[] zzUnpackRowMap() {
+ int[] result = new int[50];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
}
- private static int zzUnpackRowMap(String packed, int offset, int [] result) {
- int i = 0; /* index in packed string */
- int j = offset; /* index in unpacked array */
+ private static int zzUnpackRowMap(String packed, int offset, int[] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int high = packed.charAt(i++) << 16;
@@ -176,74 +162,72 @@ class ClassicTokenizerImpl {
return j;
}
- /**
- * The transition table of the DFA
- */
- private static final int [] ZZ_TRANS = zzUnpackTrans();
+ /** The transition table of the DFA */
+ private static final int[] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
- "\1\2\1\3\1\4\7\2\1\5\1\6\15\0\2\3"+
- "\1\0\1\7\1\0\1\10\2\11\1\12\1\3\2\0"+
- "\1\3\1\4\1\0\1\13\1\0\1\10\2\14\1\15"+
- "\1\4\2\0\1\3\1\4\1\16\1\17\1\20\1\21"+
- "\2\11\1\12\1\22\2\0\1\23\1\24\7\0\1\25"+
- "\2\0\2\26\7\0\1\26\2\0\1\27\1\30\7\0"+
- "\1\31\3\0\1\32\7\0\1\12\2\0\1\33\1\34"+
- "\7\0\1\35\2\0\1\36\1\37\7\0\1\40\2\0"+
- "\1\41\1\42\7\0\1\43\13\0\1\44\2\0\1\23"+
- "\1\24\7\0\1\45\13\0\1\46\2\0\2\26\7\0"+
- "\1\47\2\0\1\3\1\4\1\16\1\7\1\20\1\21"+
- "\2\11\1\12\1\22\2\0\2\23\1\0\1\50\1\0"+
- "\1\10\2\51\1\0\1\23\2\0\1\23\1\24\1\0"+
- "\1\52\1\0\1\10\2\53\1\54\1\24\2\0\1\23"+
- "\1\24\1\0\1\50\1\0\1\10\2\51\1\0\1\25"+
- "\2\0\2\26\1\0\1\55\2\0\1\55\2\0\1\26"+
- "\2\0\2\27\1\0\1\51\1\0\1\10\2\51\1\0"+
- "\1\27\2\0\1\27\1\30\1\0\1\53\1\0\1\10"+
- "\2\53\1\54\1\30\2\0\1\27\1\30\1\0\1\51"+
- "\1\0\1\10\2\51\1\0\1\31\3\0\1\32\1\0"+
- "\1\54\2\0\3\54\1\32\2\0\2\33\1\0\1\56"+
- "\1\0\1\10\2\11\1\12\1\33\2\0\1\33\1\34"+
- "\1\0\1\57\1\0\1\10\2\14\1\15\1\34\2\0"+
- "\1\33\1\34\1\0\1\56\1\0\1\10\2\11\1\12"+
- "\1\35\2\0\2\36\1\0\1\11\1\0\1\10\2\11"+
- "\1\12\1\36\2\0\1\36\1\37\1\0\1\14\1\0"+
- "\1\10\2\14\1\15\1\37\2\0\1\36\1\37\1\0"+
- "\1\11\1\0\1\10\2\11\1\12\1\40\2\0\2\41"+
- "\1\0\1\12\2\0\3\12\1\41\2\0\1\41\1\42"+
- "\1\0\1\15\2\0\3\15\1\42\2\0\1\41\1\42"+
- "\1\0\1\12\2\0\3\12\1\43\4\0\1\16\6\0"+
- "\1\44\2\0\1\23\1\24\1\0\1\60\1\0\1\10"+
- "\2\51\1\0\1\25\2\0\2\26\1\0\1\55\2\0"+
- "\1\55\2\0\1\47\2\0\2\23\7\0\1\23\2\0"+
- "\2\27\7\0\1\27\2\0\2\33\7\0\1\33\2\0"+
- "\2\36\7\0\1\36\2\0\2\41\7\0\1\41\2\0"+
- "\2\61\7\0\1\61\2\0\2\23\7\0\1\62\2\0"+
- "\2\61\1\0\1\55\2\0\1\55\2\0\1\61\2\0"+
- "\2\23\1\0\1\60\1\0\1\10\2\51\1\0\1\23"+
- "\1\0";
-
- private static int [] zzUnpackTrans() {
- int [] result = new int[552];
+ "\1\2\1\3\1\4\7\2\1\5\1\6\15\0\2\3"
+ + "\1\0\1\7\1\0\1\10\2\11\1\12\1\3\2\0"
+ + "\1\3\1\4\1\0\1\13\1\0\1\10\2\14\1\15"
+ + "\1\4\2\0\1\3\1\4\1\16\1\17\1\20\1\21"
+ + "\2\11\1\12\1\22\2\0\1\23\1\24\7\0\1\25"
+ + "\2\0\2\26\7\0\1\26\2\0\1\27\1\30\7\0"
+ + "\1\31\3\0\1\32\7\0\1\12\2\0\1\33\1\34"
+ + "\7\0\1\35\2\0\1\36\1\37\7\0\1\40\2\0"
+ + "\1\41\1\42\7\0\1\43\13\0\1\44\2\0\1\23"
+ + "\1\24\7\0\1\45\13\0\1\46\2\0\2\26\7\0"
+ + "\1\47\2\0\1\3\1\4\1\16\1\7\1\20\1\21"
+ + "\2\11\1\12\1\22\2\0\2\23\1\0\1\50\1\0"
+ + "\1\10\2\51\1\0\1\23\2\0\1\23\1\24\1\0"
+ + "\1\52\1\0\1\10\2\53\1\54\1\24\2\0\1\23"
+ + "\1\24\1\0\1\50\1\0\1\10\2\51\1\0\1\25"
+ + "\2\0\2\26\1\0\1\55\2\0\1\55\2\0\1\26"
+ + "\2\0\2\27\1\0\1\51\1\0\1\10\2\51\1\0"
+ + "\1\27\2\0\1\27\1\30\1\0\1\53\1\0\1\10"
+ + "\2\53\1\54\1\30\2\0\1\27\1\30\1\0\1\51"
+ + "\1\0\1\10\2\51\1\0\1\31\3\0\1\32\1\0"
+ + "\1\54\2\0\3\54\1\32\2\0\2\33\1\0\1\56"
+ + "\1\0\1\10\2\11\1\12\1\33\2\0\1\33\1\34"
+ + "\1\0\1\57\1\0\1\10\2\14\1\15\1\34\2\0"
+ + "\1\33\1\34\1\0\1\56\1\0\1\10\2\11\1\12"
+ + "\1\35\2\0\2\36\1\0\1\11\1\0\1\10\2\11"
+ + "\1\12\1\36\2\0\1\36\1\37\1\0\1\14\1\0"
+ + "\1\10\2\14\1\15\1\37\2\0\1\36\1\37\1\0"
+ + "\1\11\1\0\1\10\2\11\1\12\1\40\2\0\2\41"
+ + "\1\0\1\12\2\0\3\12\1\41\2\0\1\41\1\42"
+ + "\1\0\1\15\2\0\3\15\1\42\2\0\1\41\1\42"
+ + "\1\0\1\12\2\0\3\12\1\43\4\0\1\16\6\0"
+ + "\1\44\2\0\1\23\1\24\1\0\1\60\1\0\1\10"
+ + "\2\51\1\0\1\25\2\0\2\26\1\0\1\55\2\0"
+ + "\1\55\2\0\1\47\2\0\2\23\7\0\1\23\2\0"
+ + "\2\27\7\0\1\27\2\0\2\33\7\0\1\33\2\0"
+ + "\2\36\7\0\1\36\2\0\2\41\7\0\1\41\2\0"
+ + "\2\61\7\0\1\61\2\0\2\23\7\0\1\62\2\0"
+ + "\2\61\1\0\1\55\2\0\1\55\2\0\1\61\2\0"
+ + "\2\23\1\0\1\60\1\0\1\10\2\51\1\0\1\23"
+ + "\1\0";
+
+ private static int[] zzUnpackTrans() {
+ int[] result = new int[552];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
}
- private static int zzUnpackTrans(String packed, int offset, int [] result) {
- int i = 0; /* index in packed string */
- int j = offset; /* index in unpacked array */
+ private static int zzUnpackTrans(String packed, int offset, int[] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
value--;
- do result[j++] = value; while (--count > 0);
+ do result[j++] = value;
+ while (--count > 0);
}
return j;
}
-
/* error codes */
private static final int ZZ_UNKNOWN_ERROR = 0;
private static final int ZZ_NO_MATCH = 1;
@@ -256,30 +240,28 @@ class ClassicTokenizerImpl {
"Error: pushback value was too large"
};
- /**
- * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
- */
- private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
+ /** ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code> */
+ private static final int[] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
- "\1\0\1\11\3\1\1\11\13\0\4\1\2\0\1\1"+
- "\1\0\17\1\1\0\1\1\3\0\5\1";
+ "\1\0\1\11\3\1\1\11\13\0\4\1\2\0\1\1" + "\1\0\17\1\1\0\1\1\3\0\5\1";
- private static int [] zzUnpackAttribute() {
- int [] result = new int[50];
+ private static int[] zzUnpackAttribute() {
+ int[] result = new int[50];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
- private static int zzUnpackAttribute(String packed, int offset, int [] result) {
- int i = 0; /* index in packed string */
- int j = offset; /* index in unpacked array */
+ private static int zzUnpackAttribute(String packed, int offset, int[] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
- do result[j++] = value; while (--count > 0);
+ do result[j++] = value;
+ while (--count > 0);
}
return j;
}
@@ -293,8 +275,9 @@ class ClassicTokenizerImpl {
/** the current lexical state */
private int zzLexicalState = YYINITIAL;
- /** this buffer contains the current text to be matched and is
- the source of the yytext() string */
+ /**
+ * this buffer contains the current text to be matched and is the source of the yytext() string
+ */
private char zzBuffer[] = new char[ZZ_BUFFERSIZE];
/** the textposition at the last accepting state */
@@ -306,8 +289,7 @@ class ClassicTokenizerImpl {
/** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
- /** endRead marks the last character in the buffer, that has been read
- from input */
+ /** endRead marks the last character in the buffer, that has been read from input */
private int zzEndRead;
/** number of newlines encountered up to the start of the matched text */
@@ -316,15 +298,10 @@ class ClassicTokenizerImpl {
/** the number of characters up to the start of the matched text */
private int yychar;
- /**
- * the number of characters from the last newline up to the start of the
- * matched text
- */
+ /** the number of characters from the last newline up to the start of the matched text */
private int yycolumn;
- /**
- * zzAtBOL == true iff the scanner is currently at the beginning of a line
- */
+ /** zzAtBOL == true iff the scanner is currently at the beginning of a line */
private boolean zzAtBOL = true;
/** zzAtEOF == true iff the scanner is at the EOF */
@@ -332,81 +309,74 @@ class ClassicTokenizerImpl {
/** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
-
- /**
- * The number of occupied positions in zzBuffer beyond zzEndRead.
- * When a lead/high surrogate has been read from the input stream
- * into the final zzBuffer position, this will have a value of 1;
+
+ /**
+ * The number of occupied positions in zzBuffer beyond zzEndRead. When a lead/high surrogate has
+ * been read from the input stream into the final zzBuffer position, this will have a value of 1;
* otherwise, it will have a value of 0.
*/
private int zzFinalHighSurrogate = 0;
/* user code: */
-public static final int ALPHANUM = ClassicTokenizer.ALPHANUM;
-public static final int APOSTROPHE = ClassicTokenizer.APOSTROPHE;
-public static final int ACRONYM = ClassicTokenizer.ACRONYM;
-public static final int COMPANY = ClassicTokenizer.COMPANY;
-public static final int EMAIL = ClassicTokenizer.EMAIL;
-public static final int HOST = ClassicTokenizer.HOST;
-public static final int NUM = ClassicTokenizer.NUM;
-public static final int CJ = ClassicTokenizer.CJ;
-public static final int ACRONYM_DEP = ClassicTokenizer.ACRONYM_DEP;
+ public static final int ALPHANUM = ClassicTokenizer.ALPHANUM;
+ public static final int APOSTROPHE = ClassicTokenizer.APOSTROPHE;
+ public static final int ACRONYM = ClassicTokenizer.ACRONYM;
+ public static final int COMPANY = ClassicTokenizer.COMPANY;
+ public static final int EMAIL = ClassicTokenizer.EMAIL;
+ public static final int HOST = ClassicTokenizer.HOST;
+ public static final int NUM = ClassicTokenizer.NUM;
+ public static final int CJ = ClassicTokenizer.CJ;
+ public static final int ACRONYM_DEP = ClassicTokenizer.ACRONYM_DEP;
-public static final String [] TOKEN_TYPES = ClassicTokenizer.TOKEN_TYPES;
+ public static final String[] TOKEN_TYPES = ClassicTokenizer.TOKEN_TYPES;
-public final int yychar()
-{
+ public final int yychar() {
return yychar;
-}
-
-/**
- * Fills CharTermAttribute with the current token text.
- */
-public final void getText(CharTermAttribute t) {
- t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
-}
+ }
- public final void setBufferSize(int numChars) {
- throw new UnsupportedOperationException();
- }
+ /** Fills CharTermAttribute with the current token text. */
+ public final void getText(CharTermAttribute t) {
+ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
+ }
+ public final void setBufferSize(int numChars) {
+ throw new UnsupportedOperationException();
+ }
/**
* Creates a new scanner
*
- * @param in the java.io.Reader to read input from.
+ * @param in the java.io.Reader to read input from.
*/
ClassicTokenizerImpl(java.io.Reader in) {
this.zzReader = in;
}
-
- /**
+ /**
* Unpacks the compressed character translation table.
*
- * @param packed the packed character translation table
- * @return the unpacked character translation table
+ * @param packed the packed character translation table
+ * @return the unpacked character translation table
*/
- private static char [] zzUnpackCMap(String packed) {
- char [] map = new char[0x110000];
- int i = 0; /* index in packed string */
- int j = 0; /* index in unpacked array */
+ private static char[] zzUnpackCMap(String packed) {
+ char[] map = new char[0x110000];
+ int i = 0; /* index in packed string */
+ int j = 0; /* index in unpacked array */
while (i < 1170) {
- int count = packed.charAt(i++);
+ int count = packed.charAt(i++);
char value = packed.charAt(i++);
- do map[j++] = value; while (--count > 0);
+ do map[j++] = value;
+ while (--count > 0);
}
return map;
}
-
/**
* Refills the input buffer.
*
- * @return <code>false</code>, iff there was new input.
- *
- * @exception java.io.IOException if any I/O-Error occurs
+ * @return <code>false</code>, iff there was new input.
+ * @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
@@ -414,21 +384,19 @@ public final void getText(CharTermAttribute t) {
if (zzStartRead > 0) {
zzEndRead += zzFinalHighSurrogate;
zzFinalHighSurrogate = 0;
- System.arraycopy(zzBuffer, zzStartRead,
- zzBuffer, 0,
- zzEndRead-zzStartRead);
+ System.arraycopy(zzBuffer, zzStartRead, zzBuffer, 0, zzEndRead - zzStartRead);
/* translate stored positions */
- zzEndRead-= zzStartRead;
- zzCurrentPos-= zzStartRead;
- zzMarkedPos-= zzStartRead;
+ zzEndRead -= zzStartRead;
+ zzCurrentPos -= zzStartRead;
+ zzMarkedPos -= zzStartRead;
zzStartRead = 0;
}
/* is the buffer big enough? */
if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) {
/* if not: blow it up */
- char newBuffer[] = new char[zzBuffer.length*2];
+ char newBuffer[] = new char[zzBuffer.length * 2];
System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
zzBuffer = newBuffer;
zzEndRead += zzFinalHighSurrogate;
@@ -441,13 +409,14 @@ public final void getText(CharTermAttribute t) {
/* not supposed to occur according to specification of java.io.Reader */
if (numRead == 0) {
- throw new java.io.IOException("Reader returned 0 characters. See JFlex examples for workaround.");
+ throw new java.io.IOException(
+ "Reader returned 0 characters. See JFlex examples for workaround.");
}
if (numRead > 0) {
zzEndRead += numRead;
/* If numRead == requested, we might have requested to few chars to
- encode a full Unicode character. We assume that a Reader would
- otherwise never return half characters. */
+ encode a full Unicode character. We assume that a Reader would
+ otherwise never return half characters. */
if (numRead == requested) {
if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) {
--zzEndRead;
@@ -462,54 +431,42 @@ public final void getText(CharTermAttribute t) {
return true;
}
-
- /**
- * Closes the input stream.
- */
+ /** Closes the input stream. */
public final void yyclose() throws java.io.IOException {
- zzAtEOF = true; /* indicate end of file */
- zzEndRead = zzStartRead; /* invalidate buffer */
+ zzAtEOF = true; /* indicate end of file */
+ zzEndRead = zzStartRead; /* invalidate buffer */
- if (zzReader != null)
- zzReader.close();
+ if (zzReader != null) zzReader.close();
}
-
/**
- * Resets the scanner to read from a new input stream.
- * Does not close the old reader.
+ * Resets the scanner to read from a new input stream. Does not close the old reader.
*
- * All internal variables are reset, the old input stream
- * <b>cannot</b> be reused (internal buffer is discarded and lost).
- * Lexical state is set to <code>ZZ_INITIAL</code>.
+ * <p>All internal variables are reset, the old input stream <b>cannot</b> be reused (internal
+ * buffer is discarded and lost). Lexical state is set to <code>ZZ_INITIAL</code>.
*
- * Internal scan buffer is resized down to its initial length, if it has grown.
+ * <p>Internal scan buffer is resized down to its initial length, if it has grown.
*
- * @param reader the new input stream
+ * @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) {
zzReader = reader;
- zzAtBOL = true;
- zzAtEOF = false;
+ zzAtBOL = true;
+ zzAtEOF = false;
zzEOFDone = false;
zzEndRead = zzStartRead = 0;
zzCurrentPos = zzMarkedPos = 0;
zzFinalHighSurrogate = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
- if (zzBuffer.length > ZZ_BUFFERSIZE)
- zzBuffer = new char[ZZ_BUFFERSIZE];
+ if (zzBuffer.length > ZZ_BUFFERSIZE) zzBuffer = new char[ZZ_BUFFERSIZE];
}
-
- /**
- * Returns the current lexical state.
- */
+ /** Returns the current lexical state. */
public final int yystate() {
return zzLexicalState;
}
-
/**
* Enters a new lexical state
*
@@ -519,88 +476,71 @@ public final void getText(CharTermAttribute t) {
zzLexicalState = newState;
}
-
- /**
- * Returns the text matched by the current regular expression.
- */
+ /** Returns the text matched by the current regular expression. */
public final String yytext() {
- return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
+ return new String(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
}
-
/**
- * Returns the character at position <code>pos</code> from the
- * matched text.
- *
- * It is equivalent to yytext().charAt(pos), but faster
+ * Returns the character at position <code>pos</code> from the matched text.
*
- * @param pos the position of the character to fetch.
- * A value from 0 to yylength()-1.
+ * <p>It is equivalent to yytext().charAt(pos), but faster
*
+ * @param pos the position of the character to fetch. A value from 0 to yylength()-1.
* @return the character at position pos
*/
public final char yycharat(int pos) {
- return zzBuffer[zzStartRead+pos];
+ return zzBuffer[zzStartRead + pos];
}
-
- /**
- * Returns the length of the matched text region.
- */
+ /** Returns the length of the matched text region. */
public final int yylength() {
- return zzMarkedPos-zzStartRead;
+ return zzMarkedPos - zzStartRead;
}
-
/**
* Reports an error that occured while scanning.
*
- * In a wellformed scanner (no or only correct usage of
- * yypushback(int) and a match-all fallback rule) this method
- * will only be called with things that "Can't Possibly Happen".
- * If this method is called, something is seriously wrong
- * (e.g. a JFlex bug producing a faulty scanner etc.).
+ * <p>In a wellformed scanner (no or only correct usage of yypushback(int) and a match-all
+ * fallback rule) this method will only be called with things that "Can't Possibly Happen". If
+ * this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty
+ * scanner etc.).
*
- * Usual syntax/scanner level error handling should be done
- * in error fallback rules.
+ * <p>Usual syntax/scanner level error handling should be done in error fallback rules.
*
- * @param errorCode the code of the errormessage to display
+ * @param errorCode the code of the errormessage to display
*/
private void zzScanError(int errorCode) {
String message;
try {
message = ZZ_ERROR_MSG[errorCode];
- }
- catch (ArrayIndexOutOfBoundsException e) {
+ } catch (ArrayIndexOutOfBoundsException e) {
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
throw new Error(message);
- }
-
+ }
/**
* Pushes the specified amount of characters back into the input stream.
*
- * They will be read again by then next call of the scanning method
+ * <p>They will be read again by then next call of the scanning method
*
- * @param number the number of characters to be read again.
- * This number must not be greater than yylength()!
+ * @param number the number of characters to be read again. This number must not be greater than
+ * yylength()!
*/
- public void yypushback(int number) {
- if ( number > yylength() )
- zzScanError(ZZ_PUSHBACK_2BIG);
+ public void yypushback(int number) {
+ if (number > yylength()) zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
-
/**
- * Resumes scanning until the next regular expression is matched,
- * the end of input is encountered or an I/O-Error occurs.
+ * Resumes scanning until the next regular expression is matched, the end of input is encountered
+ * or an I/O-Error occurs.
*
- * @return the next token
- * @exception java.io.IOException if any I/O-Error occurs
+ * @return the next token
+ * @exception java.io.IOException if any I/O-Error occurs
*/
public int getNextToken() throws java.io.IOException {
int zzInput;
@@ -610,72 +550,68 @@ public final void getText(CharTermAttribute t) {
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
- char [] zzBufferL = zzBuffer;
- char [] zzCMapL = ZZ_CMAP;
+ char[] zzBufferL = zzBuffer;
+ char[] zzCMapL = ZZ_CMAP;
- int [] zzTransL = ZZ_TRANS;
- int [] zzRowMapL = ZZ_ROWMAP;
- int [] zzAttrL = ZZ_ATTRIBUTE;
+ int[] zzTransL = ZZ_TRANS;
+ int[] zzRowMapL = ZZ_ROWMAP;
+ int[] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
- yychar+= zzMarkedPosL-zzStartRead;
+ yychar += zzMarkedPosL - zzStartRead;
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
-
+
zzState = ZZ_LEXSTATE[zzLexicalState];
// set up zzAction for empty match case:
int zzAttributes = zzAttrL[zzState];
- if ( (zzAttributes & 1) == 1 ) {
+ if ((zzAttributes & 1) == 1) {
zzAction = zzState;
}
-
- zzForAction: {
+ zzForAction:
+ {
while (true) {
-
+
if (zzCurrentPosL < zzEndReadL) {
zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
zzCurrentPosL += Character.charCount(zzInput);
- }
- else if (zzAtEOF) {
+ } else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
- }
- else {
+ } else {
// store back cached positions
- zzCurrentPos = zzCurrentPosL;
- zzMarkedPos = zzMarkedPosL;
+ zzCurrentPos = zzCurrentPosL;
+ zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
- zzCurrentPosL = zzCurrentPos;
- zzMarkedPosL = zzMarkedPos;
- zzBufferL = zzBuffer;
- zzEndReadL = zzEndRead;
+ zzCurrentPosL = zzCurrentPos;
+ zzMarkedPosL = zzMarkedPos;
+ zzBufferL = zzBuffer;
+ zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
- }
- else {
+ } else {
zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
zzCurrentPosL += Character.charCount(zzInput);
}
}
- int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
+ int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
zzAttributes = zzAttrL[zzState];
- if ( (zzAttributes & 1) == 1 ) {
+ if ((zzAttributes & 1) == 1) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
- if ( (zzAttributes & 8) == 8 ) break zzForAction;
+ if ((zzAttributes & 8) == 8) break zzForAction;
}
-
}
}
@@ -685,65 +621,83 @@ public final void getText(CharTermAttribute t) {
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
return YYEOF;
- }
- else {
+ } else {
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
- case 1:
- { /* Break so we don't hit fall-through warning: */ break;/* ignore */
- }
+ case 1:
+ {
+ /* Break so we don't hit fall-through warning: */
+ break; /* ignore */
+ }
// fall through
- case 11: break;
- case 2:
- { return ALPHANUM;
- }
+ case 11:
+ break;
+ case 2:
+ {
+ return ALPHANUM;
+ }
// fall through
- case 12: break;
- case 3:
- { return CJ;
- }
+ case 12:
+ break;
+ case 3:
+ {
+ return CJ;
+ }
// fall through
- case 13: break;
- case 4:
- { return HOST;
- }
+ case 13:
+ break;
+ case 4:
+ {
+ return HOST;
+ }
// fall through
- case 14: break;
- case 5:
- { return NUM;
- }
+ case 14:
+ break;
+ case 5:
+ {
+ return NUM;
+ }
// fall through
- case 15: break;
- case 6:
- { return APOSTROPHE;
- }
+ case 15:
+ break;
+ case 6:
+ {
+ return APOSTROPHE;
+ }
// fall through
- case 16: break;
- case 7:
- { return COMPANY;
- }
+ case 16:
+ break;
+ case 7:
+ {
+ return COMPANY;
+ }
// fall through
- case 17: break;
- case 8:
- { return ACRONYM_DEP;
- }
+ case 17:
+ break;
+ case 8:
+ {
+ return ACRONYM_DEP;
+ }
// fall through
- case 18: break;
- case 9:
- { return ACRONYM;
- }
+ case 18:
+ break;
+ case 9:
+ {
+ return ACRONYM;
+ }
// fall through
- case 19: break;
- case 10:
- { return EMAIL;
- }
+ case 19:
+ break;
+ case 10:
+ {
+ return EMAIL;
+ }
// fall through
- case 20: break;
+ case 20:
+ break;
default:
zzScanError(ZZ_NO_MATCH);
}
}
}
}
-
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex
index 07d7857..798d9a5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.analysis.standard;
+package org.apache.lucene.analysis.classic;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/package-info.java
index 4b2c471..47dc52c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/package-info.java
@@ -16,14 +16,13 @@
*/
/**
- * Fast, general-purpose grammar-based tokenizers.
- * {@link org.apache.lucene.analysis.classic.ClassicTokenizer ClassicTokenizer}:
- * this class was formerly (prior to Lucene 3.1) named
- * <code>StandardTokenizer</code>. (Its tokenization rules are not
- * based on the Unicode Text Segmentation algorithm.)
- * {@link org.apache.lucene.analysis.classic.ClassicAnalyzer ClassicAnalyzer} includes
- * {@link org.apache.lucene.analysis.classic.ClassicTokenizer ClassicTokenizer},
- * {@link org.apache.lucene.analysis.LowerCaseFilter LowerCaseFilter}
- * and {@link org.apache.lucene.analysis.StopFilter StopFilter}.
+ * Fast, general-purpose grammar-based tokenizers. {@link
+ * org.apache.lucene.analysis.classic.ClassicTokenizer ClassicTokenizer}: this class was formerly
+ * (prior to Lucene 3.1) named <code>StandardTokenizer</code>. (Its tokenization rules are not based
+ * on the Unicode Text Segmentation algorithm.) {@link
+ * org.apache.lucene.analysis.classic.ClassicAnalyzer ClassicAnalyzer} includes {@link
+ * org.apache.lucene.analysis.classic.ClassicTokenizer ClassicTokenizer}, {@link
+ * org.apache.lucene.analysis.LowerCaseFilter LowerCaseFilter} and {@link
+ * org.apache.lucene.analysis.StopFilter StopFilter}.
*/
-package org.apache.lucene.analysis.classic;
\ No newline at end of file
+package org.apache.lucene.analysis.classic;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
index f32b8c0..f4c2f86 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
@@ -17,7 +17,6 @@
package org.apache.lucene.analysis.commongrams;
import java.io.IOException;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -28,20 +27,20 @@ import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
/*
- * TODO: Consider implementing https://issues.apache.org/jira/browse/LUCENE-1688 changes to stop list and associated constructors
+ * TODO: Consider implementing https://issues.apache.org/jira/browse/LUCENE-1688 changes to stop list and associated constructors
*/
/**
- * Construct bigrams for frequently occurring terms while indexing. Single terms
- * are still indexed too, with bigrams overlaid. This is achieved through the
- * use of {@link PositionIncrementAttribute#setPositionIncrement(int)}. Bigrams have a type
- * of {@link #GRAM_TYPE} Example:
+ * Construct bigrams for frequently occurring terms while indexing. Single terms are still indexed
+ * too, with bigrams overlaid. This is achieved through the use of {@link
+ * PositionIncrementAttribute#setPositionIncrement(int)}. Bigrams have a type of {@link #GRAM_TYPE}
+ * Example:
+ *
* <ul>
- * <li>input:"the quick brown fox"</li>
- * <li>output:|"the","the-quick"|"brown"|"fox"|</li>
- * <li>"the-quick" has a position increment of 0 so it is in the same position
- * as "the" "the-quick" has a term.type() of "gram"</li>
- *
+ * <li>input:"the quick brown fox"
+ * <li>output:|"the","the-quick"|"brown"|"fox"|
+ * <li>"the-quick" has a position increment of 0 so it is in the same position as "the"
+ * "the-quick" has a term.type() of "gram"
* </ul>
*/
@@ -56,23 +55,24 @@ public final class CommonGramsFilter extends TokenFilter {
private final CharArraySet commonWords;
private final StringBuilder buffer = new StringBuilder();
-
+
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
- private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
- private final PositionLengthAttribute posLenAttribute = addAttribute(PositionLengthAttribute.class);
+ private final PositionIncrementAttribute posIncAttribute =
+ addAttribute(PositionIncrementAttribute.class);
+ private final PositionLengthAttribute posLenAttribute =
+ addAttribute(PositionLengthAttribute.class);
private int lastStartOffset;
private boolean lastWasCommon;
private State savedState;
/**
- * Construct a token stream filtering the given input using a Set of common
- * words to create bigrams. Outputs both unigrams with position increment and
- * bigrams with position increment 0 type=gram where one or both of the words
- * in a potential bigram are in the set of common words .
- *
+ * Construct a token stream filtering the given input using a Set of common words to create
+ * bigrams. Outputs both unigrams with position increment and bigrams with position increment 0
+ * type=gram where one or both of the words in a potential bigram are in the set of common words .
+ *
* @param input TokenStream input in filter chain
* @param commonWords The set of common words.
*/
@@ -82,20 +82,17 @@ public final class CommonGramsFilter extends TokenFilter {
}
/**
- * Inserts bigrams for common words into a token stream. For each input token,
- * output the token. If the token and/or the following token are in the list
- * of common words also output a bigram with position increment 0 and
- * type="gram"
+ * Inserts bigrams for common words into a token stream. For each input token, output the token.
+ * If the token and/or the following token are in the list of common words also output a bigram
+ * with position increment 0 and type="gram"
*
- * TODO:Consider adding an option to not emit unigram stopwords
- * as in CDL XTF BigramStopFilter, CommonGramsQueryFilter would need to be
- * changed to work with this.
+ * <p>TODO:Consider adding an option to not emit unigram stopwords as in CDL XTF BigramStopFilter,
+ * CommonGramsQueryFilter would need to be changed to work with this.
*
- * TODO: Consider optimizing for the case of three
- * commongrams i.e "man of the year" normally produces 3 bigrams: "man-of",
- * "of-the", "the-year" but with proper management of positions we could
- * eliminate the middle bigram "of-the"and save a disk seek and a whole set of
- * position lookups.
+ * <p>TODO: Consider optimizing for the case of three commongrams i.e "man of the year" normally
+ * produces 3 bigrams: "man-of", "of-the", "the-year" but with proper management of positions we
+ * could eliminate the middle bigram "of-the"and save a disk seek and a whole set of position
+ * lookups.
*/
@Override
public boolean incrementToken() throws IOException {
@@ -108,15 +105,15 @@ public final class CommonGramsFilter extends TokenFilter {
} else if (!input.incrementToken()) {
return false;
}
-
- /* We build n-grams before and after stopwords.
+
+ /* We build n-grams before and after stopwords.
* When valid, the buffer always contains at least the separator.
* If it's empty, there is nothing before this stopword.
*/
if (lastWasCommon || (isCommon() && buffer.length() > 0)) {
savedState = captureState();
gramToken();
- return true;
+ return true;
}
saveTermBuffer();
@@ -131,20 +128,17 @@ public final class CommonGramsFilter extends TokenFilter {
buffer.setLength(0);
}
- // ================================================= Helper Methods ================================================
-
/**
* Determines if the current token is a common term
*
* @return {@code true} if the current token is a common term, {@code false} otherwise
*/
private boolean isCommon() {
- return commonWords != null && commonWords.contains(termAttribute.buffer(), 0, termAttribute.length());
+ return commonWords != null
+ && commonWords.contains(termAttribute.buffer(), 0, termAttribute.length());
}
- /**
- * Saves this information to form the left part of a gram
- */
+ /** Saves this information to form the left part of a gram */
private void saveTermBuffer() {
buffer.setLength(0);
buffer.append(termAttribute.buffer(), 0, termAttribute.length());
@@ -153,9 +147,7 @@ public final class CommonGramsFilter extends TokenFilter {
lastWasCommon = isCommon();
}
- /**
- * Constructs a compound token.
- */
+ /** Constructs a compound token. */
private void gramToken() {
buffer.append(termAttribute.buffer(), 0, termAttribute.length());
int endOffset = offsetAttribute.endOffset();
@@ -167,7 +159,7 @@ public final class CommonGramsFilter extends TokenFilter {
if (length > termText.length) {
termText = termAttribute.resizeBuffer(length);
}
-
+
buffer.getChars(0, length, termText, 0);
termAttribute.setLength(length);
posIncAttribute.setPositionIncrement(0);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilterFactory.java
index a181f4a..ccf26bd 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilterFactory.java
@@ -16,20 +16,19 @@
*/
package org.apache.lucene.analysis.commongrams;
-
import java.io.IOException;
import java.util.Map;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenFilterFactory;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
-import org.apache.lucene.analysis.TokenFilterFactory;
/**
* Constructs a {@link CommonGramsFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_cmmngrms" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -51,9 +50,9 @@ public class CommonGramsFilterFactory extends TokenFilterFactory implements Reso
private final String commonWordFiles;
private final String format;
private final boolean ignoreCase;
-
+
/** Creates a new CommonGramsFilterFactory */
- public CommonGramsFilterFactory(Map<String,String> args) {
+ public CommonGramsFilterFactory(Map<String, String> args) {
super(args);
commonWordFiles = get(args, "words");
format = get(args, "format");
@@ -95,6 +94,3 @@ public class CommonGramsFilterFactory extends TokenFilterFactory implements Reso
return commonGrams;
}
}
-
-
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
index e8c98b7..80a6381 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilter.java
@@ -16,26 +16,25 @@
*/
package org.apache.lucene.analysis.commongrams;
-import java.io.IOException;
+import static org.apache.lucene.analysis.commongrams.CommonGramsFilter.GRAM_TYPE;
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import static org.apache.lucene.analysis.commongrams.CommonGramsFilter.GRAM_TYPE;
-
/**
- * Wrap a CommonGramsFilter optimizing phrase queries by only returning single
- * words when they are not a member of a bigram.
- *
- * Example:
+ * Wrap a CommonGramsFilter optimizing phrase queries by only returning single words when they are
+ * not a member of a bigram.
+ *
+ * <p>Example:
+ *
* <ul>
- * <li>query input to CommonGramsFilter: "the rain in spain falls mainly"
- * <li>output of CommomGramsFilter/input to CommonGramsQueryFilter:
- * |"the, "the-rain"|"rain" "rain-in"|"in, "in-spain"|"spain"|"falls"|"mainly"
- * <li>output of CommonGramsQueryFilter:"the-rain", "rain-in" ,"in-spain",
- * "falls", "mainly"
+ * <li>query input to CommonGramsFilter: "the rain in spain falls mainly"
+ * <li>output of CommomGramsFilter/input to CommonGramsQueryFilter: |"the, "the-rain"|"rain"
+ * "rain-in"|"in, "in-spain"|"spain"|"falls"|"mainly"
+ * <li>output of CommonGramsQueryFilter:"the-rain", "rain-in" ,"in-spain", "falls", "mainly"
* </ul>
*/
@@ -46,16 +45,18 @@ import static org.apache.lucene.analysis.commongrams.CommonGramsFilter.GRAM_TYPE
public final class CommonGramsQueryFilter extends TokenFilter {
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
- private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
- private final PositionLengthAttribute posLengthAttribute = addAttribute(PositionLengthAttribute.class);
-
+ private final PositionIncrementAttribute posIncAttribute =
+ addAttribute(PositionIncrementAttribute.class);
+ private final PositionLengthAttribute posLengthAttribute =
+ addAttribute(PositionLengthAttribute.class);
+
private State previous;
private String previousType;
private boolean exhausted;
/**
- * Constructs a new CommonGramsQueryFilter based on the provided CommomGramsFilter
- *
+ * Constructs a new CommonGramsQueryFilter based on the provided CommomGramsFilter
+ *
* @param input CommonGramsFilter the QueryFilter will use
*/
public CommonGramsQueryFilter(CommonGramsFilter input) {
@@ -69,13 +70,14 @@ public final class CommonGramsQueryFilter extends TokenFilter {
previousType = null;
exhausted = false;
}
-
+
/**
- * Output bigrams whenever possible to optimize queries. Only output unigrams
- * when they are not a member of a bigram. Example:
+ * Output bigrams whenever possible to optimize queries. Only output unigrams when they are not a
+ * member of a bigram. Example:
+ *
* <ul>
- * <li>input: "the rain in spain falls mainly"
- * <li>output:"the-rain", "rain-in" ,"in-spain", "falls", "mainly"
+ * <li>input: "the rain in spain falls mainly"
+ * <li>output:"the-rain", "rain-in" ,"in-spain", "falls", "mainly"
* </ul>
*/
@Override
@@ -87,10 +89,11 @@ public final class CommonGramsQueryFilter extends TokenFilter {
restoreState(previous);
previous = current;
previousType = typeAttribute.type();
-
+
if (isGramType()) {
posIncAttribute.setPositionIncrement(1);
- // We must set this back to 1 (from e.g. 2 or higher) otherwise the token graph is disconnected:
+ // We must set this back to 1 (from e.g. 2 or higher) otherwise the token graph is
+ // disconnected:
posLengthAttribute.setPositionLength(1);
}
return true;
@@ -104,23 +107,22 @@ public final class CommonGramsQueryFilter extends TokenFilter {
if (previous == null || GRAM_TYPE.equals(previousType)) {
return false;
}
-
+
restoreState(previous);
previous = null;
-
+
if (isGramType()) {
posIncAttribute.setPositionIncrement(1);
- // We must set this back to 1 (from e.g. 2 or higher) otherwise the token graph is disconnected:
+ // We must set this back to 1 (from e.g. 2 or higher) otherwise the token graph is
+ // disconnected:
posLengthAttribute.setPositionLength(1);
}
return true;
}
- // ================================================= Helper Methods ================================================
-
/**
* Convenience method to check if the current type is a gram type
- *
+ *
* @return {@code true} if the current type is a gram type, {@code false} otherwise
*/
public boolean isGramType() {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilterFactory.java
index f8e712e..0e19d78 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsQueryFilterFactory.java
@@ -16,15 +16,13 @@
*/
package org.apache.lucene.analysis.commongrams;
-
import java.util.Map;
-
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
/**
* Construct {@link CommonGramsQueryFilter}.
- *
+ *
* <pre class="prettyprint">
* <fieldType name="text_cmmngrmsqry" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -41,7 +39,7 @@ public class CommonGramsQueryFilterFactory extends CommonGramsFilterFactory {
public static final String NAME = "commonGramsQuery";
/** Creates a new CommonGramsQueryFilterFactory */
- public CommonGramsQueryFilterFactory(Map<String,String> args) {
+ public CommonGramsQueryFilterFactory(Map<String, String> args) {
super(args);
}
@@ -50,9 +48,7 @@ public class CommonGramsQueryFilterFactory extends CommonGramsFilterFactory {
throw defaultCtorException();
}
- /**
- * Create a CommonGramsFilter and wrap it with a CommonGramsQueryFilter
- */
+ /** Create a CommonGramsFilter and wrap it with a CommonGramsQueryFilter */
@Override
public TokenFilter create(TokenStream input) {
CommonGramsFilter commonGrams = (CommonGramsFilter) super.create(input);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/package-info.java
index 4c9eaba..ba72f50 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/commongrams/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * Construct n-grams for frequently occurring terms and phrases.
- */
+/** Construct n-grams for frequently occurring terms and phrases. */
package org.apache.lucene.analysis.commongrams;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
index 680e67a..bc5f4b8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.analysis.compound;
-
import java.io.IOException;
import java.util.LinkedList;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -27,23 +25,15 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-/**
- * Base class for decomposition token filters.
- */
+/** Base class for decomposition token filters. */
public abstract class CompoundWordTokenFilterBase extends TokenFilter {
- /**
- * The default for minimal word length that gets decomposed
- */
+ /** The default for minimal word length that gets decomposed */
public static final int DEFAULT_MIN_WORD_SIZE = 5;
- /**
- * The default for minimal length of subwords that get propagated to the output of this filter
- */
+ /** The default for minimal length of subwords that get propagated to the output of this filter */
public static final int DEFAULT_MIN_SUBWORD_SIZE = 2;
- /**
- * The default for maximal length of subwords that get propagated to the output of this filter
- */
+ /** The default for maximal length of subwords that get propagated to the output of this filter */
public static final int DEFAULT_MAX_SUBWORD_SIZE = 15;
protected final CharArraySet dictionary;
@@ -55,34 +45,54 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
protected final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionIncrementAttribute posIncAtt =
+ addAttribute(PositionIncrementAttribute.class);
private State current;
- protected CompoundWordTokenFilterBase(TokenStream input, CharArraySet dictionary, boolean onlyLongestMatch) {
- this(input,dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, onlyLongestMatch);
+ protected CompoundWordTokenFilterBase(
+ TokenStream input, CharArraySet dictionary, boolean onlyLongestMatch) {
+ this(
+ input,
+ dictionary,
+ DEFAULT_MIN_WORD_SIZE,
+ DEFAULT_MIN_SUBWORD_SIZE,
+ DEFAULT_MAX_SUBWORD_SIZE,
+ onlyLongestMatch);
}
protected CompoundWordTokenFilterBase(TokenStream input, CharArraySet dictionary) {
- this(input,dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, false);
+ this(
+ input,
+ dictionary,
+ DEFAULT_MIN_WORD_SIZE,
+ DEFAULT_MIN_SUBWORD_SIZE,
+ DEFAULT_MAX_SUBWORD_SIZE,
+ false);
}
- protected CompoundWordTokenFilterBase(TokenStream input, CharArraySet dictionary, int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
+ protected CompoundWordTokenFilterBase(
+ TokenStream input,
+ CharArraySet dictionary,
+ int minWordSize,
+ int minSubwordSize,
+ int maxSubwordSize,
+ boolean onlyLongestMatch) {
super(input);
- this.tokens=new LinkedList<>();
+ this.tokens = new LinkedList<>();
if (minWordSize < 0) {
throw new IllegalArgumentException("minWordSize cannot be negative");
}
- this.minWordSize=minWordSize;
+ this.minWordSize = minWordSize;
if (minSubwordSize < 0) {
throw new IllegalArgumentException("minSubwordSize cannot be negative");
}
- this.minSubwordSize=minSubwordSize;
+ this.minSubwordSize = minSubwordSize;
if (maxSubwordSize < 0) {
throw new IllegalArgumentException("maxSubwordSize cannot be negative");
}
- this.maxSubwordSize=maxSubwordSize;
- this.onlyLongestMatch=onlyLongestMatch;
+ this.maxSubwordSize = maxSubwordSize;
+ this.onlyLongestMatch = onlyLongestMatch;
this.dictionary = dictionary;
}
@@ -115,8 +125,10 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
}
}
- /** Decomposes the current {@link #termAtt} and places {@link CompoundToken} instances in the {@link #tokens} list.
- * The original token may not be placed in the list, as it is automatically passed through this filter.
+ /**
+ * Decomposes the current {@link #termAtt} and places {@link CompoundToken} instances in the
+ * {@link #tokens} list. The original token may not be placed in the list, as it is automatically
+ * passed through this filter.
*/
protected abstract void decompose();
@@ -127,21 +139,21 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
current = null;
}
- /**
- * Helper class to hold decompounded token information
- */
+ /** Helper class to hold decompounded token information */
protected class CompoundToken {
public final CharSequence txt;
public final int startOffset, endOffset;
- /** Construct the compound token based on a slice of the current {@link CompoundWordTokenFilterBase#termAtt}. */
+ /**
+ * Construct the compound token based on a slice of the current {@link
+ * CompoundWordTokenFilterBase#termAtt}.
+ */
public CompoundToken(int offset, int length) {
this.txt = CompoundWordTokenFilterBase.this.termAtt.subSequence(offset, offset + length);
-
+
// offsets of the original word
this.startOffset = CompoundWordTokenFilterBase.this.offsetAtt.startOffset();
this.endOffset = CompoundWordTokenFilterBase.this.offsetAtt.endOffset();
}
-
- }
+ }
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
index 2e4b837..c6278a8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
@@ -16,27 +16,23 @@
*/
package org.apache.lucene.analysis.compound;
-
-
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.TokenStream;
/**
- * A {@link org.apache.lucene.analysis.TokenFilter} that decomposes compound words found in many Germanic languages.
- * <p>
- * "Donaudampfschiff" becomes Donau, dampf, schiff so that you can find
- * "Donaudampfschiff" even when you only enter "schiff".
- * It uses a brute-force algorithm to achieve this.
+ * A {@link org.apache.lucene.analysis.TokenFilter} that decomposes compound words found in many
+ * Germanic languages.
+ *
+ * <p>"Donaudampfschiff" becomes Donau, dampf, schiff so that you can find "Donaudampfschiff" even
+ * when you only enter "schiff". It uses a brute-force algorithm to achieve this.
*/
public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBase {
/**
* Creates a new {@link DictionaryCompoundWordTokenFilter}
*
- * @param input
- * the {@link org.apache.lucene.analysis.TokenStream} to process
- * @param dictionary
- * the word dictionary to match against.
+ * @param input the {@link org.apache.lucene.analysis.TokenStream} to process
+ * @param dictionary the word dictionary to match against.
*/
public DictionaryCompoundWordTokenFilter(TokenStream input, CharArraySet dictionary) {
super(input, dictionary);
@@ -48,21 +44,20 @@ public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBa
/**
* Creates a new {@link DictionaryCompoundWordTokenFilter}
*
- * @param input
- * the {@link org.apache.lucene.analysis.TokenStream} to process
- * @param dictionary
- * the word dictionary to match against.
- * @param minWordSize
- * only words longer than this get processed
- * @param minSubwordSize
- * only subwords longer than this get to the output stream
- * @param maxSubwordSize
- * only subwords shorter than this get to the output stream
- * @param onlyLongestMatch
- * Add only the longest matching subword to the stream
+ * @param input the {@link org.apache.lucene.analysis.TokenStream} to process
+ * @param dictionary the word dictionary to match against.
+ * @param minWordSize only words longer than this get processed
+ * @param minSubwordSize only subwords longer than this get to the output stream
+ * @param maxSubwordSize only subwords shorter than this get to the output stream
+ * @param onlyLongestMatch Add only the longest matching subword to the stream
*/
- public DictionaryCompoundWordTokenFilter(TokenStream input, CharArraySet dictionary,
- int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
+ public DictionaryCompoundWordTokenFilter(
+ TokenStream input,
+ CharArraySet dictionary,
+ int minWordSize,
+ int minSubwordSize,
+ int maxSubwordSize,
+ boolean onlyLongestMatch) {
super(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
if (dictionary == null) {
throw new IllegalArgumentException("dictionary must not be null");
@@ -72,29 +67,29 @@ public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBa
@Override
protected void decompose() {
final int len = termAtt.length();
- for (int i=0;i<=len-this.minSubwordSize;++i) {
- CompoundToken longestMatchToken=null;
- for (int j=this.minSubwordSize;j<=this.maxSubwordSize;++j) {
- if(i+j>len) {
- break;
- }
- if(dictionary.contains(termAtt.buffer(), i, j)) {
- if (this.onlyLongestMatch) {
- if (longestMatchToken!=null) {
- if (longestMatchToken.txt.length()<j) {
- longestMatchToken=new CompoundToken(i,j);
- }
- } else {
- longestMatchToken=new CompoundToken(i,j);
- }
- } else {
- tokens.add(new CompoundToken(i,j));
- }
- }
+ for (int i = 0; i <= len - this.minSubwordSize; ++i) {
+ CompoundToken longestMatchToken = null;
+ for (int j = this.minSubwordSize; j <= this.maxSubwordSize; ++j) {
+ if (i + j > len) {
+ break;
}
- if (this.onlyLongestMatch && longestMatchToken!=null) {
- tokens.add(longestMatchToken);
+ if (dictionary.contains(termAtt.buffer(), i, j)) {
+ if (this.onlyLongestMatch) {
+ if (longestMatchToken != null) {
+ if (longestMatchToken.txt.length() < j) {
+ longestMatchToken = new CompoundToken(i, j);
+ }
+ } else {
+ longestMatchToken = new CompoundToken(i, j);
+ }
+ } else {
+ tokens.add(new CompoundToken(i, j));
+ }
}
+ }
+ if (this.onlyLongestMatch && longestMatchToken != null) {
+ tokens.add(longestMatchToken);
+ }
}
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java
index 347a5ba..6981973 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java
@@ -16,18 +16,17 @@
*/
package org.apache.lucene.analysis.compound;
-
import java.io.IOException;
import java.util.Map;
-
import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.TokenFilterFactory;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
-import org.apache.lucene.analysis.TokenFilterFactory;
-/**
+/**
* Factory for {@link DictionaryCompoundWordTokenFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_dictcomp" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -40,7 +39,8 @@ import org.apache.lucene.analysis.TokenFilterFactory;
* @since 3.1
* @lucene.spi {@value #NAME}
*/
-public class DictionaryCompoundWordTokenFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
+public class DictionaryCompoundWordTokenFilterFactory extends TokenFilterFactory
+ implements ResourceLoaderAware {
/** SPI name */
public static final String NAME = "dictionaryCompoundWord";
@@ -57,14 +57,16 @@ public class DictionaryCompoundWordTokenFilterFactory extends TokenFilterFactory
super(args);
dictFile = require(args, "dictionary");
minWordSize = getInt(args, "minWordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE);
- minSubwordSize = getInt(args, "minSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
- maxSubwordSize = getInt(args, "maxSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
+ minSubwordSize =
+ getInt(args, "minSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
+ maxSubwordSize =
+ getInt(args, "maxSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
onlyLongestMatch = getBoolean(args, "onlyLongestMatch", true);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public DictionaryCompoundWordTokenFilterFactory() {
throw defaultCtorException();
@@ -74,14 +76,14 @@ public class DictionaryCompoundWordTokenFilterFactory extends TokenFilterFactory
public void inform(ResourceLoader loader) throws IOException {
dictionary = super.getWordSet(loader, dictFile, false);
}
-
+
@Override
public TokenStream create(TokenStream input) {
// if the dictionary is null, it means it was empty
if (dictionary == null) {
return input;
}
- return new DictionaryCompoundWordTokenFilter(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
+ return new DictionaryCompoundWordTokenFilter(
+ input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
}
}
-
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
index 41f92c9..3acb5c2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
@@ -17,7 +17,6 @@
package org.apache.lucene.analysis.compound;
import java.io.IOException;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.compound.hyphenation.Hyphenation;
@@ -25,83 +24,91 @@ import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
import org.xml.sax.InputSource;
/**
- * A {@link org.apache.lucene.analysis.TokenFilter} that decomposes compound words found in many Germanic languages.
+ * A {@link org.apache.lucene.analysis.TokenFilter} that decomposes compound words found in many
+ * Germanic languages.
*
- * "Donaudampfschiff" becomes Donau, dampf, schiff so that you can find
- * "Donaudampfschiff" even when you only enter "schiff". It uses a hyphenation
- * grammar and a word dictionary to achieve this.
+ * <p>"Donaudampfschiff" becomes Donau, dampf, schiff so that you can find "Donaudampfschiff" even
+ * when you only enter "schiff". It uses a hyphenation grammar and a word dictionary to achieve
+ * this.
*/
-public class HyphenationCompoundWordTokenFilter extends
- CompoundWordTokenFilterBase {
+public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterBase {
private HyphenationTree hyphenator;
/**
* Creates a new {@link HyphenationCompoundWordTokenFilter} instance.
*
- * @param input
- * the {@link org.apache.lucene.analysis.TokenStream} to process
- * @param hyphenator
- * the hyphenation pattern tree to use for hyphenation
- * @param dictionary
- * the word dictionary to match against.
+ * @param input the {@link org.apache.lucene.analysis.TokenStream} to process
+ * @param hyphenator the hyphenation pattern tree to use for hyphenation
+ * @param dictionary the word dictionary to match against.
*/
- public HyphenationCompoundWordTokenFilter(TokenStream input,
- HyphenationTree hyphenator, CharArraySet dictionary) {
- this(input, hyphenator, dictionary, DEFAULT_MIN_WORD_SIZE,
- DEFAULT_MIN_SUBWORD_SIZE, DEFAULT_MAX_SUBWORD_SIZE, false);
+ public HyphenationCompoundWordTokenFilter(
+ TokenStream input, HyphenationTree hyphenator, CharArraySet dictionary) {
+ this(
+ input,
+ hyphenator,
+ dictionary,
+ DEFAULT_MIN_WORD_SIZE,
+ DEFAULT_MIN_SUBWORD_SIZE,
+ DEFAULT_MAX_SUBWORD_SIZE,
+ false);
}
/**
* Creates a new {@link HyphenationCompoundWordTokenFilter} instance.
*
- * @param input
- * the {@link org.apache.lucene.analysis.TokenStream} to process
- * @param hyphenator
- * the hyphenation pattern tree to use for hyphenation
- * @param dictionary
- * the word dictionary to match against.
- * @param minWordSize
- * only words longer than this get processed
- * @param minSubwordSize
- * only subwords longer than this get to the output stream
- * @param maxSubwordSize
- * only subwords shorter than this get to the output stream
- * @param onlyLongestMatch
- * Add only the longest matching subword to the stream
+ * @param input the {@link org.apache.lucene.analysis.TokenStream} to process
+ * @param hyphenator the hyphenation pattern tree to use for hyphenation
+ * @param dictionary the word dictionary to match against.
+ * @param minWordSize only words longer than this get processed
+ * @param minSubwordSize only subwords longer than this get to the output stream
+ * @param maxSubwordSize only subwords shorter than this get to the output stream
+ * @param onlyLongestMatch Add only the longest matching subword to the stream
*/
- public HyphenationCompoundWordTokenFilter(TokenStream input,
- HyphenationTree hyphenator, CharArraySet dictionary, int minWordSize,
- int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
- super(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize,
- onlyLongestMatch);
+ public HyphenationCompoundWordTokenFilter(
+ TokenStream input,
+ HyphenationTree hyphenator,
+ CharArraySet dictionary,
+ int minWordSize,
+ int minSubwordSize,
+ int maxSubwordSize,
+ boolean onlyLongestMatch) {
+ super(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
this.hyphenator = hyphenator;
}
/**
* Create a HyphenationCompoundWordTokenFilter with no dictionary.
- * <p>
- * Calls {@link #HyphenationCompoundWordTokenFilter(org.apache.lucene.analysis.TokenStream, org.apache.lucene.analysis.compound.hyphenation.HyphenationTree, org.apache.lucene.analysis.CharArraySet, int, int, int, boolean)
- * HyphenationCompoundWordTokenFilter(matchVersion, input, hyphenator,
- * null, minWordSize, minSubwordSize, maxSubwordSize }
+ *
+ * <p>Calls {@link #HyphenationCompoundWordTokenFilter(org.apache.lucene.analysis.TokenStream,
+ * org.apache.lucene.analysis.compound.hyphenation.HyphenationTree,
+ * org.apache.lucene.analysis.CharArraySet, int, int, int, boolean)
+ * HyphenationCompoundWordTokenFilter(matchVersion, input, hyphenator, null, minWordSize,
+ * minSubwordSize, maxSubwordSize }
*/
- public HyphenationCompoundWordTokenFilter(TokenStream input,
- HyphenationTree hyphenator, int minWordSize, int minSubwordSize,
- int maxSubwordSize) {
- this(input, hyphenator, null, minWordSize, minSubwordSize,
- maxSubwordSize, false);
+ public HyphenationCompoundWordTokenFilter(
+ TokenStream input,
+ HyphenationTree hyphenator,
+ int minWordSize,
+ int minSubwordSize,
+ int maxSubwordSize) {
+ this(input, hyphenator, null, minWordSize, minSubwordSize, maxSubwordSize, false);
}
/**
* Create a HyphenationCompoundWordTokenFilter with no dictionary.
- * <p>
- * Calls {@link #HyphenationCompoundWordTokenFilter(org.apache.lucene.analysis.TokenStream, org.apache.lucene.analysis.compound.hyphenation.HyphenationTree, int, int, int)
- * HyphenationCompoundWordTokenFilter(matchVersion, input, hyphenator,
- * DEFAULT_MIN_WORD_SIZE, DEFAULT_MIN_SUBWORD_SIZE, DEFAULT_MAX_SUBWORD_SIZE }
+ *
+ * <p>Calls {@link #HyphenationCompoundWordTokenFilter(org.apache.lucene.analysis.TokenStream,
+ * org.apache.lucene.analysis.compound.hyphenation.HyphenationTree, int, int, int)
+ * HyphenationCompoundWordTokenFilter(matchVersion, input, hyphenator, DEFAULT_MIN_WORD_SIZE,
+ * DEFAULT_MIN_SUBWORD_SIZE, DEFAULT_MAX_SUBWORD_SIZE }
*/
- public HyphenationCompoundWordTokenFilter(TokenStream input,
- HyphenationTree hyphenator) {
- this(input, hyphenator, DEFAULT_MIN_WORD_SIZE, DEFAULT_MIN_SUBWORD_SIZE,
+ public HyphenationCompoundWordTokenFilter(TokenStream input, HyphenationTree hyphenator) {
+ this(
+ input,
+ hyphenator,
+ DEFAULT_MIN_WORD_SIZE,
+ DEFAULT_MIN_SUBWORD_SIZE,
DEFAULT_MAX_SUBWORD_SIZE);
}
@@ -112,8 +119,7 @@ public class HyphenationCompoundWordTokenFilter extends
* @return An object representing the hyphenation patterns
* @throws java.io.IOException If there is a low-level I/O error.
*/
- public static HyphenationTree getHyphenationTree(String hyphenationFilename)
- throws IOException {
+ public static HyphenationTree getHyphenationTree(String hyphenationFilename) throws IOException {
return getHyphenationTree(new InputSource(hyphenationFilename));
}
@@ -158,7 +164,7 @@ public class HyphenationCompoundWordTokenFilter extends
// we only put subwords to the token stream
// that are longer than minPartSize
if (partLength < this.minSubwordSize) {
- // BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the
+ // BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the
// calculation above, and we rely upon minSubwordSize being >=0 to filter them out...
continue;
}
@@ -194,7 +200,7 @@ public class HyphenationCompoundWordTokenFilter extends
}
}
}
- if (this.onlyLongestMatch && longestMatchToken!=null) {
+ if (this.onlyLongestMatch && longestMatchToken != null) {
tokens.add(longestMatchToken);
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java
index 2d70346..a1f8809 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java
@@ -16,37 +16,39 @@
*/
package org.apache.lucene.analysis.compound;
-
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
-
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenFilterFactory;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
-import org.apache.lucene.analysis.TokenFilterFactory;
-import org.apache.lucene.util.IOUtils;
import org.xml.sax.InputSource;
/**
* Factory for {@link HyphenationCompoundWordTokenFilter}.
- * <p>
- * This factory accepts the following parameters:
+ *
+ * <p>This factory accepts the following parameters:
+ *
* <ul>
- * <li><code>hyphenator</code> (mandatory): path to the FOP xml hyphenation pattern.
- * See <a href="http://offo.sourceforge.net/hyphenation/">http://offo.sourceforge.net/hyphenation/</a>.
- * <li><code>encoding</code> (optional): encoding of the xml hyphenation file. defaults to UTF-8.
- * <li><code>dictionary</code> (optional): dictionary of words. defaults to no dictionary.
- * <li><code>minWordSize</code> (optional): minimal word length that gets decomposed. defaults to 5.
- * <li><code>minSubwordSize</code> (optional): minimum length of subwords. defaults to 2.
- * <li><code>maxSubwordSize</code> (optional): maximum length of subwords. defaults to 15.
- * <li><code>onlyLongestMatch</code> (optional): if true, adds only the longest matching subword
- * to the stream. defaults to false.
+ * <li><code>hyphenator</code> (mandatory): path to the FOP xml hyphenation pattern. See <a
+ * href="http://offo.sourceforge.net/hyphenation/">http://offo.sourceforge.net/hyphenation/</a>.
+ * <li><code>encoding</code> (optional): encoding of the xml hyphenation file. defaults to UTF-8.
+ * <li><code>dictionary</code> (optional): dictionary of words. defaults to no dictionary.
+ * <li><code>minWordSize</code> (optional): minimal word length that gets decomposed. defaults to
+ * 5.
+ * <li><code>minSubwordSize</code> (optional): minimum length of subwords. defaults to 2.
+ * <li><code>maxSubwordSize</code> (optional): maximum length of subwords. defaults to 15.
+ * <li><code>onlyLongestMatch</code> (optional): if true, adds only the longest matching subword
+ * to the stream. defaults to false.
* </ul>
+ *
* <br>
+ *
* <pre class="prettyprint">
* <fieldType name="text_hyphncomp" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
@@ -60,7 +62,8 @@ import org.xml.sax.InputSource;
* @since 3.1.0
* @lucene.spi {@value #NAME}
*/
-public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
+public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactory
+ implements ResourceLoaderAware {
/** SPI name */
public static final String NAME = "hyphenationCompoundWord";
@@ -74,7 +77,7 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactor
private final int minSubwordSize;
private final int maxSubwordSize;
private final boolean onlyLongestMatch;
-
+
/** Creates a new HyphenationCompoundWordTokenFilterFactory */
public HyphenationCompoundWordTokenFilterFactory(Map<String, String> args) {
super(args);
@@ -82,14 +85,16 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactor
encoding = get(args, "encoding");
hypFile = require(args, "hyphenator");
minWordSize = getInt(args, "minWordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE);
- minSubwordSize = getInt(args, "minSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
- maxSubwordSize = getInt(args, "maxSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
+ minSubwordSize =
+ getInt(args, "minSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
+ maxSubwordSize =
+ getInt(args, "maxSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
onlyLongestMatch = getBoolean(args, "onlyLongestMatch", false);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
-
+
/** Default ctor for compatibility with SPI */
public HyphenationCompoundWordTokenFilterFactory() {
throw defaultCtorException();
@@ -100,7 +105,7 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactor
InputStream stream = null;
try {
if (dictFile != null) // the dictionary can be empty.
- dictionary = getWordSet(loader, dictFile, false);
+ dictionary = getWordSet(loader, dictFile, false);
// TODO: Broken, because we cannot resolve real system id
// ResourceLoader should also supply method like ClassLoader to get resource URL
stream = loader.openResource(hypFile);
@@ -112,9 +117,16 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactor
IOUtils.closeWhileHandlingException(stream);
}
}
-
+
@Override
public TokenFilter create(TokenStream input) {
- return new HyphenationCompoundWordTokenFilter(input, hyphenator, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
+ return new HyphenationCompoundWordTokenFilter(
+ input,
+ hyphenator,
+ dictionary,
+ minWordSize,
+ minSubwordSize,
+ maxSubwordSize,
+ onlyLongestMatch);
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java
index 8e83893..6531bac 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/ByteVector.java
@@ -18,27 +18,21 @@
package org.apache.lucene.analysis.compound.hyphenation;
/**
- * This class implements a simple byte vector with access to the underlying
- * array.
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * This class implements a simple byte vector with access to the underlying array. This class has
+ * been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been
+ * slightly modified.
*/
public class ByteVector {
- /**
- * Capacity increment size
- */
+ /** Capacity increment size */
private static final int DEFAULT_BLOCK_SIZE = 2048;
private int blockSize;
- /**
- * The encapsulated array
- */
+ /** The encapsulated array */
private byte[] array;
- /**
- * Points to next free item
- */
+ /** Points to next free item */
private int n;
public ByteVector() {
@@ -75,16 +69,12 @@ public class ByteVector {
return array;
}
- /**
- * return number of items in array
- */
+ /** return number of items in array */
public int length() {
return n;
}
- /**
- * returns current capacity of array
- */
+ /** returns current capacity of array */
public int capacity() {
return array.length;
}
@@ -97,9 +87,7 @@ public class ByteVector {
return array[index];
}
- /**
- * This is to implement memory allocation in the array. Like malloc().
- */
+ /** This is to implement memory allocation in the array. Like malloc(). */
public int alloc(int size) {
int index = n;
int len = array.length;
@@ -119,5 +107,4 @@ public class ByteVector {
array = aux;
}
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
index c61ac19..1161b8b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
@@ -17,28 +17,22 @@
package org.apache.lucene.analysis.compound.hyphenation;
/**
- * This class implements a simple char vector with access to the underlying
- * array.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * This class implements a simple char vector with access to the underlying array.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
public class CharVector implements Cloneable {
- /**
- * Capacity increment size
- */
+ /** Capacity increment size */
private static final int DEFAULT_BLOCK_SIZE = 2048;
private int blockSize;
- /**
- * The encapsulated array
- */
+ /** The encapsulated array */
private char[] array;
- /**
- * Points to next free item
- */
+ /** Points to next free item */
private int n;
public CharVector() {
@@ -71,9 +65,7 @@ public class CharVector implements Cloneable {
n = a.length;
}
- /**
- * Reset Vector but don't resize or clear elements
- */
+ /** Reset Vector but don't resize or clear elements */
public void clear() {
n = 0;
}
@@ -89,16 +81,12 @@ public class CharVector implements Cloneable {
return array;
}
- /**
- * return number of items in array
- */
+ /** return number of items in array */
public int length() {
return n;
}
- /**
- * returns current capacity of array
- */
+ /** returns current capacity of array */
public int capacity() {
return array.length;
}
@@ -130,5 +118,4 @@ public class CharVector implements Cloneable {
array = aux;
}
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
index 1845f25..db92ce2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
@@ -17,18 +17,16 @@
package org.apache.lucene.analysis.compound.hyphenation;
/**
- * This class represents a hyphen. A 'full' hyphen is made of 3 parts: the
- * pre-break text, post-break text and no-break. If no line-break is generated
- * at this position, the no-break text is used, otherwise, pre-break and
- * post-break are used. Typically, pre-break is equal to the hyphen character
- * and the others are empty. However, this general scheme allows support for
- * cases in some languages where words change spelling if they're split across
- * lines, like german's 'backen' which hyphenates 'bak-ken'. BTW, this comes
- * from TeX.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * This class represents a hyphen. A 'full' hyphen is made of 3 parts: the pre-break text,
+ * post-break text and no-break. If no line-break is generated at this position, the no-break text
+ * is used, otherwise, pre-break and post-break are used. Typically, pre-break is equal to the
+ * hyphen character and the others are empty. However, this general scheme allows support for cases
+ * in some languages where words change spelling if they're split across lines, like german's
+ * 'backen' which hyphenates 'bak-ken'. BTW, this comes from TeX.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
-
public class Hyphen {
public String preBreak;
@@ -50,8 +48,7 @@ public class Hyphen {
@Override
public String toString() {
- if (noBreak == null && postBreak == null && preBreak != null
- && preBreak.equals("-")) {
+ if (noBreak == null && postBreak == null && preBreak != null && preBreak.equals("-")) {
return "-";
}
StringBuilder res = new StringBuilder("{");
@@ -63,5 +60,4 @@ public class Hyphen {
res.append('}');
return res.toString();
}
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphenation.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphenation.java
index 3fb1e04..6b28ea5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphenation.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphenation.java
@@ -18,30 +18,25 @@ package org.apache.lucene.analysis.compound.hyphenation;
/**
* This class represents a hyphenated word.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
public class Hyphenation {
private int[] hyphenPoints;
- /**
- * rawWord as made of alternating strings and {@link Hyphen Hyphen} instances
- */
+ /** rawWord as made of alternating strings and {@link Hyphen Hyphen} instances */
Hyphenation(int[] points) {
hyphenPoints = points;
}
- /**
- * @return the number of hyphenation points in the word
- */
+ /** @return the number of hyphenation points in the word */
public int length() {
return hyphenPoints.length;
}
- /**
- * @return the hyphenation points
- */
+ /** @return the hyphenation points */
public int[] getHyphenationPoints() {
return hyphenPoints;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
index 70ad08e..acc0955 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
@@ -20,35 +20,27 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
-
import org.xml.sax.InputSource;
/**
- * This tree structure stores the hyphenation patterns in an efficient way for
- * fast lookup. It provides the provides the method to hyphenate a word.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * This tree structure stores the hyphenation patterns in an efficient way for fast lookup. It
+ * provides the provides the method to hyphenate a word.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
public class HyphenationTree extends TernaryTree implements PatternConsumer {
- /**
- * value space: stores the interletter values
- */
+ /** value space: stores the interletter values */
protected ByteVector vspace;
- /**
- * This map stores hyphenation exceptions
- */
- protected HashMap<String,ArrayList<Object>> stoplist;
+ /** This map stores hyphenation exceptions */
+ protected HashMap<String, ArrayList<Object>> stoplist;
- /**
- * This map stores the character classes
- */
+ /** This map stores the character classes */
protected TernaryTree classmap;
- /**
- * Temporary map to store interletter values on pattern loading.
- */
+ /** Temporary map to store interletter values on pattern loading. */
private transient TernaryTree ivalues;
public HyphenationTree() {
@@ -59,12 +51,10 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
}
/**
- * Packs the values by storing them in 4 bits, two values into a byte Values
- * range is from 0 to 9. We use zero as terminator, so we'll add 1 to the
- * value.
- *
- * @param values a string of digits from '0' to '9' representing the
- * interletter values.
+ * Packs the values by storing them in 4 bits, two values into a byte Values range is from 0 to 9.
+ * We use zero as terminator, so we'll add 1 to the value.
+ *
+ * @param values a string of digits from '0' to '9' representing the interletter values.
* @return the index into the vspace array where the packed values are stored.
*/
protected int packValues(String values) {
@@ -89,7 +79,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
StringBuilder buf = new StringBuilder();
byte v = vspace.get(k++);
while (v != 0) {
- char c = (char) (((v & 0xf0 )>>> 4) - 1 + '0');
+ char c = (char) (((v & 0xf0) >>> 4) - 1 + '0');
buf.append(c);
c = (char) (v & 0x0f);
if (c == 0) {
@@ -104,7 +94,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
/**
* Read hyphenation patterns from an XML file.
- *
+ *
* @param source the InputSource for the file
* @throws IOException In case the parsing fails
*/
@@ -132,9 +122,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
return "";
}
- /**
- * String compare, returns 0 if equal or t is a substring of s
- */
+ /** String compare, returns 0 if equal or t is a substring of s */
protected int hstrcmp(char[] s, int si, char[] t, int ti) {
for (; s[si] == t[ti]; si++, ti++) {
if (s[si] == 0) {
@@ -151,7 +139,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
StringBuilder buf = new StringBuilder();
byte v = vspace.get(k++);
while (v != 0) {
- char c = (char) (((v & 0xf0 )>>> 4) - 1);
+ char c = (char) (((v & 0xf0) >>> 4) - 1);
buf.append(c);
c = (char) (v & 0x0f);
if (c == 0) {
@@ -169,27 +157,21 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
}
/**
- * <p>
- * Search for all possible partial matches of word starting at index an update
- * interletter values. In other words, it does something like:
- * </p>
- * <code>
+ * Search for all possible partial matches of word starting at index an update interletter values.
+ * In other words, it does something like: <code>
* for(i=0; i<patterns.length; i++) {
* if ( word.substring(index).startsWidth(patterns[i]) )
* update_interletter_values(patterns[i]);
* }
* </code>
- * <p>
- * But it is done in an efficient way since the patterns are stored in a
- * ternary tree. In fact, this is the whole purpose of having the tree: doing
- * this search without having to test every single pattern. The number of
- * patterns for languages such as English range from 4000 to 10000. Thus,
- * doing thousands of string comparisons for each word to hyphenate would be
- * really slow without the tree. The tradeoff is memory, but using a ternary
- * tree instead of a trie, almost halves the memory used by Lout or TeX.
- * It's also faster than using a hash table
- * </p>
- *
+ *
+ * <p>But it is done in an efficient way since the patterns are stored in a ternary tree. In fact,
+ * this is the whole purpose of having the tree: doing this search without having to test every
+ * single pattern. The number of patterns for languages such as English range from 4000 to 10000.
+ * Thus, doing thousands of string comparisons for each word to hyphenate would be really slow
+ * without the tree. The tradeoff is memory, but using a ternary tree instead of a trie, almost
+ * halves the memory used by Lout or TeX. It's also faster than using a hash table
+ *
* @param word null terminated word to match
* @param index start index from word
* @param il interletter values array to update
@@ -244,8 +226,8 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
q = lo[q];
/**
- * actually the code should be: q = sc[q] < 0 ? hi[q] : lo[q]; but
- * java chars are unsigned
+ * actually the code should be: q = sc[q] < 0 ? hi[q] : lo[q]; but java chars are
+ * unsigned
*/
}
}
@@ -257,50 +239,42 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
/**
* Hyphenate word and return a Hyphenation object.
- *
+ *
* @param word the word to be hyphenated
- * @param remainCharCount Minimum number of characters allowed before the
- * hyphenation point.
- * @param pushCharCount Minimum number of characters allowed after the
- * hyphenation point.
- * @return a {@link Hyphenation Hyphenation} object representing the
- * hyphenated word or null if word is not hyphenated.
+ * @param remainCharCount Minimum number of characters allowed before the hyphenation point.
+ * @param pushCharCount Minimum number of characters allowed after the hyphenation point.
+ * @return a {@link Hyphenation Hyphenation} object representing the hyphenated word or null if
+ * word is not hyphenated.
*/
- public Hyphenation hyphenate(String word, int remainCharCount,
- int pushCharCount) {
+ public Hyphenation hyphenate(String word, int remainCharCount, int pushCharCount) {
char[] w = word.toCharArray();
return hyphenate(w, 0, w.length, remainCharCount, pushCharCount);
}
/**
- * w = "****nnllllllnnn*****", where n is a non-letter, l is a letter, all n
- * may be absent, the first n is at offset, the first l is at offset +
- * iIgnoreAtBeginning; word = ".llllll.'\0'***", where all l in w are copied
- * into word. In the first part of the routine len = w.length, in the second
- * part of the routine len = word.length. Three indices are used: index(w),
- * the index in w, index(word), the index in word, letterindex(word), the
- * index in the letter part of word. The following relations exist: index(w) =
- * offset + i - 1 index(word) = i - iIgnoreAtBeginning letterindex(word) =
- * index(word) - 1 (see first loop). It follows that: index(w) - index(word) =
- * offset - 1 + iIgnoreAtBeginning index(w) = letterindex(word) + offset +
- * iIgnoreAtBeginning
+ * w = "****nnllllllnnn*****", where n is a non-letter, l is a letter, all n may be absent, the
+ * first n is at offset, the first l is at offset + iIgnoreAtBeginning; word = ".llllll.'\0'***",
+ * where all l in w are copied into word. In the first part of the routine len = w.length, in the
+ * second part of the routine len = word.length. Three indices are used: index(w), the index in w,
+ * index(word), the index in word, letterindex(word), the index in the letter part of word. The
+ * following relations exist: index(w) = offset + i - 1 index(word) = i - iIgnoreAtBeginning
+ * letterindex(word) = index(word) - 1 (see first loop). It follows that: index(w) - index(word) =
+ * offset - 1 + iIgnoreAtBeginning index(w) = letterindex(word) + offset + iIgnoreAtBeginning
*/
/**
* Hyphenate word and return an array of hyphenation points.
- *
+ *
* @param w char array that contains the word
* @param offset Offset to first character in word
* @param len Length of word
- * @param remainCharCount Minimum number of characters allowed before the
- * hyphenation point.
- * @param pushCharCount Minimum number of characters allowed after the
- * hyphenation point.
- * @return a {@link Hyphenation Hyphenation} object representing the
- * hyphenated word or null if word is not hyphenated.
+ * @param remainCharCount Minimum number of characters allowed before the hyphenation point.
+ * @param pushCharCount Minimum number of characters allowed after the hyphenation point.
+ * @return a {@link Hyphenation Hyphenation} object representing the hyphenated word or null if
+ * word is not hyphenated.
*/
- public Hyphenation hyphenate(char[] w, int offset, int len,
- int remainCharCount, int pushCharCount) {
+ public Hyphenation hyphenate(
+ char[] w, int offset, int len, int remainCharCount, int pushCharCount) {
int i;
char[] word = new char[len + 3];
@@ -370,8 +344,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
// i + 1 is index(word),
// result[k] = corresponding index(w)
for (i = 0; i < len; i++) {
- if (((il[i + 1] & 1) == 1) && i >= remainCharCount
- && i <= (len - pushCharCount)) {
+ if (((il[i + 1] & 1) == 1) && i >= remainCharCount && i <= (len - pushCharCount)) {
result[k++] = i + iIgnoreAtBeginning;
}
}
@@ -379,12 +352,12 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
if (k > 0) {
// trim result array
- int[] res = new int[k+2];
+ int[] res = new int[k + 2];
System.arraycopy(result, 0, res, 1, k);
// We add the synthetical hyphenation points
// at the beginning and end of the word
- res[0]=0;
- res[k+1]=len;
+ res[0] = 0;
+ res[k + 1] = len;
return new Hyphenation(res);
} else {
return null;
@@ -392,14 +365,13 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
}
/**
- * Add a character class to the tree. It is used by
- * {@link PatternParser PatternParser} as callback to add character classes.
- * Character classes define the valid word characters for hyphenation. If a
- * word contains a character not defined in any of the classes, it is not
- * hyphenated. It also defines a way to normalize the characters in order to
- * compare them with the stored patterns. Usually pattern files use only lower
- * case characters, in this case a class for letter 'a', for example, should
- * be defined as "aA", the first character being the normalization char.
+ * Add a character class to the tree. It is used by {@link PatternParser PatternParser} as
+ * callback to add character classes. Character classes define the valid word characters for
+ * hyphenation. If a word contains a character not defined in any of the classes, it is not
+ * hyphenated. It also defines a way to normalize the characters in order to compare them with the
+ * stored patterns. Usually pattern files use only lower case characters, in this case a class for
+ * letter 'a', for example, should be defined as "aA", the first character being the normalization
+ * char.
*/
@Override
public void addClass(String chargroup) {
@@ -415,13 +387,11 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
}
/**
- * Add an exception to the tree. It is used by
- * {@link PatternParser PatternParser} class as callback to store the
- * hyphenation exceptions.
- *
+ * Add an exception to the tree. It is used by {@link PatternParser PatternParser} class as
+ * callback to store the hyphenation exceptions.
+ *
* @param word normalized word
- * @param hyphenatedword a vector of alternating strings and
- * {@link Hyphen hyphen} objects.
+ * @param hyphenatedword a vector of alternating strings and {@link Hyphen hyphen} objects.
*/
@Override
public void addException(String word, ArrayList<Object> hyphenatedword) {
@@ -429,14 +399,13 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
}
/**
- * Add a pattern to the tree. Mainly, to be used by
- * {@link PatternParser PatternParser} class as callback to add a pattern to
- * the tree.
- *
+ * Add a pattern to the tree. Mainly, to be used by {@link PatternParser PatternParser} class as
+ * callback to add a pattern to the tree.
+ *
* @param pattern the hyphenation pattern
- * @param ivalue interletter weight values indicating the desirability and
- * priority of hyphenating at a given point within the pattern. It
- * should contain only digit characters. (i.e. '0' to '9').
+ * @param ivalue interletter weight values indicating the desirability and priority of hyphenating
+ * at a given point within the pattern. It should contain only digit characters. (i.e. '0' to
+ * '9').
*/
@Override
public void addPattern(String pattern, String ivalue) {
@@ -450,9 +419,7 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
@Override
public void printStats(PrintStream out) {
- out.println("Value space size = "
- + Integer.toString(vspace.length()));
+ out.println("Value space size = " + Integer.toString(vspace.length()));
super.printStats(out);
-
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternConsumer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternConsumer.java
index 358841c..abe5763 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternConsumer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternConsumer.java
@@ -19,36 +19,33 @@ package org.apache.lucene.analysis.compound.hyphenation;
import java.util.ArrayList;
/**
- * This interface is used to connect the XML pattern file parser to the
- * hyphenation tree.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * This interface is used to connect the XML pattern file parser to the hyphenation tree.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
public interface PatternConsumer {
/**
- * Add a character class. A character class defines characters that are
- * considered equivalent for the purpose of hyphenation (e.g. "aA"). It
- * usually means to ignore case.
- *
+ * Add a character class. A character class defines characters that are considered equivalent for
+ * the purpose of hyphenation (e.g. "aA"). It usually means to ignore case.
+ *
* @param chargroup character group
*/
void addClass(String chargroup);
/**
- * Add a hyphenation exception. An exception replaces the result obtained by
- * the algorithm for cases for which this fails or the user wants to provide
- * his own hyphenation. A hyphenatedword is a vector of alternating String's
- * and {@link Hyphen Hyphen} instances
+ * Add a hyphenation exception. An exception replaces the result obtained by the algorithm for
+ * cases for which this fails or the user wants to provide his own hyphenation. A hyphenatedword
+ * is a vector of alternating String's and {@link Hyphen Hyphen} instances
*/
void addException(String word, ArrayList<Object> hyphenatedword);
/**
* Add hyphenation patterns.
- *
+ *
* @param pattern the pattern
* @param values interletter values expressed as a string of digit characters.
*/
void addPattern(String pattern, String values);
-
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
index f5f31d8..33a762b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
@@ -16,25 +16,21 @@
*/
package org.apache.lucene.analysis.compound.hyphenation;
-// SAX
-import org.xml.sax.XMLReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import javax.xml.parsers.SAXParserFactory;
+import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
+import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
-import org.xml.sax.Attributes;
-
-// Java
-import java.io.IOException;
-import java.util.ArrayList;
-
-import javax.xml.parsers.SAXParserFactory;
/**
- * A SAX document handler to read and parse hyphenation patterns from a XML
- * file.
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ * A SAX document handler to read and parse hyphenation patterns from a XML file.
+ *
+ * <p>This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/).
+ * They have been slightly modified.
*/
public class PatternParser extends DefaultHandler {
@@ -67,7 +63,6 @@ public class PatternParser extends DefaultHandler {
parser.setErrorHandler(this);
parser.setEntityResolver(this);
hyphenChar = '-'; // default
-
}
public PatternParser(PatternConsumer consumer) {
@@ -81,7 +76,7 @@ public class PatternParser extends DefaultHandler {
/**
* Parses a hyphenation pattern file.
- *
+ *
* @param filename the filename
* @throws IOException In case of an exception while parsing
*/
@@ -91,7 +86,7 @@ public class PatternParser extends DefaultHandler {
/**
* Parses a hyphenation pattern file.
- *
+ *
* @param source the InputSource for the file
* @throws IOException In case of an exception while parsing
*/
@@ -105,7 +100,7 @@ public class PatternParser extends DefaultHandler {
/**
* Creates a SAX parser using JAXP
- *
+ *
* @return the created SAX parser
*/
static XMLReader createParser() {
@@ -242,10 +237,8 @@ public class PatternParser extends DefaultHandler {
@Override
public InputSource resolveEntity(String publicId, String systemId) {
// supply the internal hyphenation.dtd if possible
- if (
- (systemId != null && systemId.matches("(?i).*\\bhyphenation.dtd\\b.*")) ||
- ("hyphenation-info".equals(publicId))
- ) {
+ if ((systemId != null && systemId.matches("(?i).*\\bhyphenation.dtd\\b.*"))
+ || ("hyphenation-info".equals(publicId))) {
// System.out.println(this.getClass().getResource("hyphenation.dtd").toExternalForm());
return new InputSource(this.getClass().getResource("hyphenation.dtd").toExternalForm());
}
@@ -257,12 +250,11 @@ public class PatternParser extends DefaultHandler {
//
/**
- * @see org.xml.sax.ContentHandler#startElement(java.lang.String,
- * java.lang.String, java.lang.String, org.xml.sax.Attributes)
+ * @see org.xml.sax.ContentHandler#startElement(java.lang.String, java.lang.String,
+ * java.lang.String, org.xml.sax.Attributes)
*/
@Override
- public void startElement(String uri, String local, String raw,
- Attributes attrs) {
+ public void startElement(String uri, String local, String raw, Attributes attrs) {
if (local.equals("hyphen-char")) {
String h = attrs.getValue("value");
if (h != null && h.length() == 1) {
@@ -279,16 +271,16 @@ public class PatternParser extends DefaultHandler {
if (token.length() > 0) {
exception.add(token.toString());
}
- exception.add(new Hyphen(attrs.getValue("pre"), attrs.getValue("no"),
- attrs.getValue("post")));
+ exception.add(
+ new Hyphen(attrs.getValue("pre"), attrs.getValue("no"), attrs.getValue("post")));
currElement = ELEM_HYPHEN;
}
token.setLength(0);
}
/**
- * @see org.xml.sax.ContentHandler#endElement(java.lang.String,
- * java.lang.String, java.lang.String)
+ * @see org.xml.sax.ContentHandler#endElement(java.lang.String, java.lang.String,
+ * java.lang.String)
*/
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
@@ -303,8 +295,7 @@ public class PatternParser extends DefaultHandler {
case ELEM_EXCEPTIONS:
exception.add(word);
exception = normalizeException(exception);
- consumer.addException(getExceptionWord(exception),
- (ArrayList) exception.clone());
+ consumer.addException(getExceptionWord(exception), (ArrayList) exception.clone());
break;
case ELEM_PATTERNS:
consumer.addPattern(getPattern(word), getInterletterValues(word));
@@ -322,12 +313,9 @@ public class PatternParser extends DefaultHandler {
} else {
currElement = 0;
}
-
}
- /**
- * @see org.xml.sax.ContentHandler#characters(char[], int, int)
- */
+ /** @see org.xml.sax.ContentHandler#characters(char[], int, int) */
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public void characters(char ch[], int start, int length) {
@@ -343,8 +331,7 @@ public class PatternParser extends DefaultHandler {
case ELEM_EXCEPTIONS:
exception.add(word);
exception = normalizeException(exception);
- consumer.addException(getExceptionWord(exception),
- (ArrayList) exception.clone());
+ consumer.addException(getExceptionWord(exception), (ArrayList) exception.clone());
exception.clear();
break;
case ELEM_PATTERNS:
@@ -353,12 +340,9 @@ public class PatternParser extends DefaultHandler {
}
word = readToken(chars);
}
-
}
- /**
- * Returns a string of the location.
- */
+ /** Returns a string of the location. */
private String getLocationString(SAXParseException ex) {
StringBuilder str = new StringBuilder();
@@ -376,6 +360,5 @@ public class PatternParser extends DefaultHandler {
str.append(ex.getColumnNumber());
return str.toString();
-
} // getLocationString(SAXParseException):String
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
index 6d1d71a..15e9ed8 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
@@ -21,92 +21,71 @@ import java.util.Enumeration;
import java.util.Stack;
/**
+ *
+ *
* <h2>Ternary Search Tree.</h2>
- *
- * <p>
- * A ternary search tree is a hybrid between a binary tree and a digital search
- * tree (trie). Keys are limited to strings. A data value of type char is stored
- * in each leaf node. It can be used as an index (or pointer) to the data.
- * Branches that only contain one key are compressed to one node by storing a
- * pointer to the trailer substring of the key. This class is intended to serve
- * as base class or helper class to implement Dictionary collections or the
- * like. Ternary trees have some nice properties as the following: the tree can
- * be traversed in sorted order, partial matches (wildcard) can be implemented,
- * retrieval of all keys within a given distance from the target, etc. The
- * storage requirements are higher than a binary tree but a lot less than a
- * trie. Performance is comparable with a hash table, sometimes it outperforms a
- * hash function (most of the time can determine a miss faster than a hash).
- * </p>
- *
- * <p>
- * The main purpose of this java port is to serve as a base for implementing
- * TeX's hyphenation algorithm (see The TeXBook, appendix H). Each language
- * requires from 5000 to 15000 hyphenation patterns which will be keys in this
- * tree. The strings patterns are usually small (from 2 to 5 characters), but
- * each char in the tree is stored in a node. Thus memory usage is the main
- * concern. We will sacrifice 'elegance' to keep memory requirements to the
- * minimum. Using java's char type as pointer (yes, I know pointer it is a
- * forbidden word in java) we can keep the size of the node to be just 8 bytes
- * (3 pointers and the data char). This gives room for about 65000 nodes. In my
- * tests the english patterns took 7694 nodes and the german patterns 10055
- * nodes, so I think we are safe.
- * </p>
- *
- * <p>
- * All said, this is a map with strings as keys and char as value. Pretty
- * limited!. It can be extended to a general map by using the string
- * representation of an object and using the char value as an index to an array
- * that contains the object values.
- * </p>
- *
- * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
+ *
+ * <p>A ternary search tree is a hybrid between a binary tree and a digital search tree (trie). Keys
+ * are limited to strings. A data value of type char is stored in each leaf node. It can be used as
+ * an index (or pointer) to the data. Branches that only contain one key are compressed to one node
+ * by storing a pointer to the trailer substring of the key. This class is intended to serve as base
+ * class or helper class to implement Dictionary collections or the like. Ternary trees have some
+ * nice properties as the following: the tree can be traversed in sorted order, partial matches
+ * (wildcard) can be implemented, retrieval of all keys within a given distance from the target,
+ * etc. The storage requirements are higher than a binary tree but a lot less than a trie.
+ * Performance is comparable with a hash table, sometimes it outperforms a hash function (most of
+ * the time can determine a miss faster than a hash).
+ *
+ * <p>The main purpose of this java port is to serve as a base for implementing TeX's hyphenation
+ * algorithm (see The TeXBook, appendix H). Each language requires from 5000 to 15000 hyphenation
+ * patterns which will be keys in this tree. The strings patterns are usually small (from 2 to 5
+ * characters), but each char in the tree is stored in a node. Thus memory usage is the main
+ * concern. We will sacrifice 'elegance' to keep memory requirements to the minimum. Using java's
+ * char type as pointer (yes, I know pointer it is a forbidden word in java) we can keep the size of
+ * the node to be just 8 bytes (3 pointers and the data char). This gives room for about 65000
+ * nodes. In my tests the english patterns took 7694 nodes and the german patterns 10055 nodes, so I
+ * think we are safe.
+ *
+ * <p>All said, this is a map with strings as keys and char as value. Pretty limited!. It can be
+ * extended to a general map by using the string representation of an object and using the char
+ * value as an index to an array that contains the object values. This class has been taken from the
+ * Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-
public class TernaryTree implements Cloneable {
/**
- * We use 4 arrays to represent a node. I guess I should have created a proper
- * node class, but somehow Knuth's pascal code made me forget we now have a
- * portable language with virtual memory management and automatic garbage
- * collection! And now is kind of late, furthermore, if it ain't broken, don't
- * fix it.
+ * We use 4 arrays to represent a node. I guess I should have created a proper node class, but
+ * somehow Knuth's pascal code made me forget we now have a portable language with virtual memory
+ * management and automatic garbage collection! And now is kind of late, furthermore, if it ain't
+ * broken, don't fix it.
*/
/**
- * Pointer to low branch and to rest of the key when it is stored directly in
- * this node, we don't have unions in java!
+ * Pointer to low branch and to rest of the key when it is stored directly in this node, we don't
+ * have unions in java!
*/
protected char[] lo;
- /**
- * Pointer to high branch.
- */
+ /** Pointer to high branch. */
protected char[] hi;
- /**
- * Pointer to equal branch and to data when this node is a string terminator.
- */
+ /** Pointer to equal branch and to data when this node is a string terminator. */
protected char[] eq;
/**
- * <P>
- * The character stored in this node: splitchar. Two special values are
- * reserved:
- * </P>
+ * The character stored in this node: splitchar. Two special values are reserved:
+ *
* <ul>
- * <li>0x0000 as string terminator</li>
- * <li>0xFFFF to indicate that the branch starting at this node is compressed</li>
+ * <li>0x0000 as string terminator
+ * <li>0xFFFF to indicate that the branch starting at this node is compressed
* </ul>
- * <p>
- * This shouldn't be a problem if we give the usual semantics to strings since
- * 0xFFFF is guaranteed not to be an Unicode character.
- * </p>
+ *
+ * <p>This shouldn't be a problem if we give the usual semantics to strings since 0xFFFF is
+ * guaranteed not to be an Unicode character.
*/
protected char[] sc;
- /**
- * This vector holds the trailing of the keys when the branch is compressed.
- */
+ /** This vector holds the trailing of the keys when the branch is compressed. */
protected CharVector kv;
protected char root;
@@ -133,10 +112,9 @@ public class TernaryTree implements Cloneable {
}
/**
- * Branches are initially compressed, needing one node per key plus the size
- * of the string key. They are decompressed as needed when another key with
- * same prefix is inserted. This saves a lot of space, specially for long
- * keys.
+ * Branches are initially compressed, needing one node per key plus the size of the string key.
+ * They are decompressed as needed when another key with same prefix is inserted. This saves a lot
+ * of space, specially for long keys.
*/
public void insert(String key, char val) {
// make sure we have enough room in the arrays
@@ -158,9 +136,7 @@ public class TernaryTree implements Cloneable {
root = insert(root, key, start, val);
}
- /**
- * The actual insertion function, recursive version.
- */
+ /** The actual insertion function, recursive version. */
private char insert(char p, char[] key, int start, char val) {
int len = strlen(key, start);
if (p == 0) {
@@ -230,9 +206,7 @@ public class TernaryTree implements Cloneable {
return p;
}
- /**
- * Compares 2 null terminated char arrays
- */
+ /** Compares 2 null terminated char arrays */
public static int strcmp(char[] a, int startA, char[] b, int startB) {
for (; a[startA] == b[startB]; startA++, startB++) {
if (a[startA] == 0) {
@@ -242,9 +216,7 @@ public class TernaryTree implements Cloneable {
return a[startA] - b[startB];
}
- /**
- * Compares a string with null terminated char array
- */
+ /** Compares a string with null terminated char array */
public static int strcmp(String str, char[] a, int start) {
int i, d, len = str.length();
for (i = 0; i < len; i++) {
@@ -260,7 +232,6 @@ public class TernaryTree implements Cloneable {
return -a[start + i];
}
return 0;
-
}
public static void strcpy(char[] dst, int di, char[] src, int si) {
@@ -363,9 +334,9 @@ public class TernaryTree implements Cloneable {
}
/**
- * Recursively insert the median first and then the median of the lower and
- * upper halves, and so on in order to get a balanced tree. The array of keys
- * is assumed to be sorted in ascending order.
+ * Recursively insert the median first and then the median of the lower and upper halves, and so
+ * on in order to get a balanced tree. The array of keys is assumed to be sorted in ascending
+ * order.
*/
protected void insertBalanced(String[] k, char[] v, int offset, int n) {
int m;
@@ -380,9 +351,7 @@ public class TernaryTree implements Cloneable {
insertBalanced(k, v, offset + m + 1, n - m - 1);
}
- /**
- * Balance the tree for best search performance
- */
+ /** Balance the tree for best search performance */
public void balance() {
// System.out.print("Before root splitchar = ");
// System.out.println(sc[root]);
@@ -404,15 +373,12 @@ public class TernaryTree implements Cloneable {
}
/**
- * Each node stores a character (splitchar) which is part of some key(s). In a
- * compressed branch (one that only contain a single string key) the trailer
- * of the key which is not already in nodes is stored externally in the kv
- * array. As items are inserted, key substrings decrease. Some substrings may
- * completely disappear when the whole branch is totally decompressed. The
- * tree is traversed to find the key substrings actually used. In addition,
- * duplicate substrings are removed using a map (implemented with a
- * TernaryTree!).
- *
+ * Each node stores a character (splitchar) which is part of some key(s). In a compressed branch
+ * (one that only contain a single string key) the trailer of the key which is not already in
+ * nodes is stored externally in the kv array. As items are inserted, key substrings decrease.
+ * Some substrings may completely disappear when the whole branch is totally decompressed. The
+ * tree is traversed to find the key substrings actually used. In addition, duplicate substrings
+ * are removed using a map (implemented with a TernaryTree!).
*/
public void trimToSize() {
// first balance the tree for best performance
@@ -458,18 +424,15 @@ public class TernaryTree implements Cloneable {
/**
* Enumeration over TST keys
+ *
* @lucene.internal
*/
public class Iterator implements Enumeration<String> {
- /**
- * current node index
- */
+ /** current node index */
int cur;
- /**
- * current key
- */
+ /** current key */
String curkey;
... 125364 lines suppressed ...
[lucene-solr] 01/03: Correct extra * in javadoc.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 9af588e71eab72e81c06422aff3fbdf12df70663
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Wed Dec 23 12:43:39 2020 +0100
Correct extra * in javadoc.
---
lucene/core/src/java/org/apache/lucene/LucenePackage.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lucene/core/src/java/org/apache/lucene/LucenePackage.java b/lucene/core/src/java/org/apache/lucene/LucenePackage.java
index 4e44edd..da903f4 100644
--- a/lucene/core/src/java/org/apache/lucene/LucenePackage.java
+++ b/lucene/core/src/java/org/apache/lucene/LucenePackage.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene;
-/** Lucene's package information, including version. * */
+/** Lucene's package information, including version. */
public final class LucenePackage {
private LucenePackage() {} // can't construct