You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2010/01/14 20:05:42 UTC
svn commit: r899359 [1/7] - in /lucene/java/branches/flex_1458: ./ contrib/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/
contrib/analyzers/common/src/java/org/apache/l...
Author: uschindler
Date: Thu Jan 14 19:05:12 2010
New Revision: 899359
URL: http://svn.apache.org/viewvc?rev=899359&view=rev
Log:
merge flex branch to trunk rev 899001
Added:
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/da_UTF8.xml
- copied unchanged from r899001, lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/da_UTF8.xml
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/hyphenation.dtd
- copied unchanged from r899001, lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/hyphenation.dtd
lucene/java/branches/flex_1458/contrib/benchmark/conf/collation.alg
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/conf/collation.alg
lucene/java/branches/flex_1458/contrib/benchmark/scripts/
- copied from r899001, lucene/java/trunk/contrib/benchmark/scripts/
lucene/java/branches/flex_1458/contrib/benchmark/scripts/collation.bm2jira.pl
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl
lucene/java/branches/flex_1458/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java
- copied unchanged from r899001, lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java
lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
- copied unchanged from r899001, lucene/java/trunk/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
- copied unchanged from r899001, lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
lucene/java/branches/flex_1458/contrib/lucli/src/test/
- copied from r899001, lucene/java/trunk/contrib/lucli/src/test/
lucene/java/branches/flex_1458/contrib/lucli/src/test/add.tests.here
- copied unchanged from r899001, lucene/java/trunk/contrib/lucli/src/test/add.tests.here
lucene/java/branches/flex_1458/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
- copied unchanged from r899001, lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/
- copied from r899001, lucene/java/trunk/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/
lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java
- copied unchanged from r899001, lucene/java/trunk/contrib/spatial/src/test/org/apache/lucene/spatial/geohash/TestGeoHashUtils.java
lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/
- copied from r899001, lucene/java/trunk/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/
lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java
- copied unchanged from r899001, lucene/java/trunk/contrib/spatial/src/test/org/apache/lucene/spatial/geometry/TestDistanceUnits.java
lucene/java/branches/flex_1458/contrib/wordnet/src/test/add.tests.here
- copied unchanged from r899001, lucene/java/trunk/contrib/wordnet/src/test/add.tests.here
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
- copied unchanged from r899001, lucene/java/trunk/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopwordAnalyzerBase.java
- copied unchanged from r899001, lucene/java/trunk/src/java/org/apache/lucene/analysis/StopwordAnalyzerBase.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/VirtualMethod.java
- copied unchanged from r899001, lucene/java/trunk/src/java/org/apache/lucene/util/VirtualMethod.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestAssertions.java
- copied unchanged from r899001, lucene/java/trunk/src/test/org/apache/lucene/TestAssertions.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/wordliststopwords.txt
- copied unchanged from r899001, lucene/java/trunk/src/test/org/apache/lucene/index/wordliststopwords.txt
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/wordliststopwords_nocomment.txt
- copied unchanged from r899001, lucene/java/trunk/src/test/org/apache/lucene/index/wordliststopwords_nocomment.txt
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestVirtualMethod.java
- copied unchanged from r899001, lucene/java/trunk/src/test/org/apache/lucene/util/TestVirtualMethod.java
Modified:
lucene/java/branches/flex_1458/ (props changed)
lucene/java/branches/flex_1458/CHANGES.txt
lucene/java/branches/flex_1458/build.xml
lucene/java/branches/flex_1458/common-build.xml
lucene/java/branches/flex_1458/contrib/ (props changed)
lucene/java/branches/flex_1458/contrib/CHANGES.txt (contents, props changed)
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/IdentityEncoder.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
lucene/java/branches/flex_1458/contrib/analyzers/smartcn/build.xml (contents, props changed)
lucene/java/branches/flex_1458/contrib/analyzers/smartcn/pom.xml.template (contents, props changed)
lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/ant/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/benchmark/CHANGES.txt
lucene/java/branches/flex_1458/contrib/benchmark/build.xml
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java (props changed)
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java
lucene/java/branches/flex_1458/contrib/benchmark/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/PerfTaskTest.java (contents, props changed)
lucene/java/branches/flex_1458/contrib/db/bdb-je/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/db/bdb/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/pom.xml.template (props changed)
lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java
lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java
lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (props changed)
lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java
lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java
lucene/java/branches/flex_1458/contrib/highlighter/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/highlighter/src/test/ (props changed)
lucene/java/branches/flex_1458/contrib/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java
lucene/java/branches/flex_1458/contrib/icu/src/java/overview.html
lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedAllTermDocs.java
lucene/java/branches/flex_1458/contrib/instantiated/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/lucli/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/memory/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/ChainedFilter.java
lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java (props changed)
lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/queryParser/ext/ExtendableQueryParser.java
lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/queryParser/ext/ExtensionQuery.java
lucene/java/branches/flex_1458/contrib/misc/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java
lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/complexPhrase/TestComplexPhraseQuery.java (props changed)
lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtensions.java
lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java
lucene/java/branches/flex_1458/contrib/queries/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/MultiFieldQueryParserWrapper.java
lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/QueryParserWrapper.java
lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
lucene/java/branches/flex_1458/contrib/regex/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/remote/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/snowball/build.xml
lucene/java/branches/flex_1458/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
lucene/java/branches/flex_1458/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
lucene/java/branches/flex_1458/contrib/snowball/src/java/org/tartarus/snowball/Among.java
lucene/java/branches/flex_1458/contrib/snowball/src/java/org/tartarus/snowball/SnowballProgram.java
lucene/java/branches/flex_1458/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/ (props changed)
lucene/java/branches/flex_1458/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashUtils.java
lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/geometry/DistanceUnits.java
lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/geometry/shape/DistanceApproximation.java
lucene/java/branches/flex_1458/contrib/spatial/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesianShapeFilter.java (props changed)
lucene/java/branches/flex_1458/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/java/branches/flex_1458/contrib/spellchecker/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/surround/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/wikipedia/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/wordnet/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/contrib/xml-query-parser/docs/LuceneContribQuery.dtd.org.html (contents, props changed)
lucene/java/branches/flex_1458/contrib/xml-query-parser/docs/LuceneCoreQuery.dtd.org.html (contents, props changed)
lucene/java/branches/flex_1458/contrib/xml-query-parser/docs/index.html (contents, props changed)
lucene/java/branches/flex_1458/contrib/xml-query-parser/src/java/overview.html (props changed)
lucene/java/branches/flex_1458/docs/lucene-contrib/index.html (contents, props changed)
lucene/java/branches/flex_1458/src/demo/org/apache/lucene/demo/html/HTMLParser.java
lucene/java/branches/flex_1458/src/demo/org/apache/lucene/demo/html/HTMLParser.jj
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Analyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/BaseCharFilter.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/CharArraySet.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/CharFilter.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/CharReader.java (contents, props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/CharStream.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/LowerCaseFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/MappingCharFilter.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NormalizeCharMap.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NativeFSLockFactory.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NoLockFactory.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/AttributeSource.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BitVector.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Parameter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/ReaderUtil.java (props changed)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleLRUCache.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/cache/SimpleMapCache.java
lucene/java/branches/flex_1458/src/site/src/documentation/skins/common/images/README.txt (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/common/translations/CommonMessages_de.xml (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/common/translations/CommonMessages_en_US.xml (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/common/translations/CommonMessages_es.xml (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/common/translations/CommonMessages_fr.xml (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/lucene/css/basic.css (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/lucene/css/print.css (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/lucene/css/screen.css (props changed)
lucene/java/branches/flex_1458/src/site/src/documentation/skins/lucene/note.txt (props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSearch.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestSnapshotDeletionPolicy.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestAnalyzers.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestCharArraySet.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java (contents, props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestAtomicUpdate.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestTransactions.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/queryParser/TestQueryParser.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/CheckHits.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/QueryUtils.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java (props changed)
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/function/FunctionTestSetup.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/function/TestFieldScoreQuery.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/function/TestOrdValues.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/store/TestLockFactory.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/English.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/LocalizedTestCase.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/_TestUtil.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/cache/TestDoubleBarrelLRUCache.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/cache/TestSimpleLRUCache.java
Propchange: lucene/java/branches/flex_1458/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Thu Jan 14 19:05:12 2010
@@ -1,5 +1,8 @@
build
dist
+backwards
+target
+test
*~
velocity.log
build.properties
Propchange: lucene/java/branches/flex_1458/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 14 19:05:12 2010
@@ -1,5 +1,5 @@
/lucene/java/branches/lucene_2_4:748824
-/lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036
+/lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036,896850
/lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
-/lucene/java/branches/lucene_3_0:880793
-/lucene/java/trunk:824912-825292,827043-833960,880727-886190,889185,889622,889667
+/lucene/java/branches/lucene_3_0:880793,896906
+/lucene/java/trunk:824912-825292,827043-833960,880727-886190,889185,889622,889667,889866-899001
Modified: lucene/java/branches/flex_1458/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/CHANGES.txt?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/CHANGES.txt (original)
+++ lucene/java/branches/flex_1458/CHANGES.txt Thu Jan 14 19:05:12 2010
@@ -12,8 +12,22 @@
FuzzyQuery. The class was never intended to be public.
(Uwe Schindler, Mike McCandless)
+* LUCENE-2135: Added FieldCache.purge(IndexReader) method to the
+ interface. Anyone implementing FieldCache externally will need to
+ fix their code to implement this, on upgrading. (Mike McCandless)
+
+* LUCENE-1923: Renamed SegmentInfo & SegmentInfos segString method to
+ toString. These are advanced APIs and subject to change suddenly.
+ (Tim Smith via Mike McCandless)
+
Changes in runtime behavior
+* LUCENE-1923: Made IndexReader.toString() produce something
+ meaningful (Tim Smith via Mike McCandless)
+
+* LUCENE-2179: CharArraySet.clear() is now functional.
+ (Robert Muir, Uwe Schindler)
+
API Changes
* LUCENE-2076: Rename FSDirectory.getFile -> getDirectory. (George
@@ -29,6 +43,25 @@
until Lucene 4.0 the default one will be deprecated.
(Shai Erera via Uwe Schindler)
+* LUCENE-1609: Restore IndexReader.getTermInfosIndexDivisor (it was
+ accidentally removed in 3.0.0) (Mike McCandless)
+
+* LUCENE-1972: Restore SortField.getComparatorSource (it was
+ accidentally removed in 3.0.0) (John Wang via Uwe Schindler)
+
+* LUCENE-2177: Deprecate the Field ctors that take byte[] and Store.
+ Since the removal of compressed fields, Store can only be YES, so
+ it's not necessary to specify. (Erik Hatcher via Mike McCandless)
+
+* LUCENE-2190: Added setNextReader method to CustomScoreQuery, which
+ is necessary with per-segment searching to notify the subclass
+ which reader the int doc, passed to customScore, refers to. (Paul
+ chez Jamespot via Mike McCandless)
+
+* LUCENE-2200: Several final classes had non-overriding protected
+ members. These were converted to private and unused protected
+ constructors removed. (Steven Rowe via Robert Muir)
+
Bug fixes
* LUCENE-2092: BooleanQuery was ignoring disableCoord in its hashCode
@@ -49,6 +82,19 @@
* LUCENE-2142: FieldCacheImpl.getStringIndex no longer throws an
exception when term count exceeds doc count. (Mike McCandless)
+
+* LUCENE-2166: Don't incorrectly keep warning about the same immense
+ term, when IndexWriter.infoStream is on. (Mike McCandless)
+
+* LUCENE-2104: NativeFSLock.release() would silently fail if the lock is held by
+ another thread/process. (Shai Erera via Uwe Schindler)
+
+* LUCENE-2158: At high indexing rates, NRT reader could temporarily
+ lose deletions. (Mike McCandless)
+
+* LUCENE-2182: DEFAULT_ATTRIBUTE_FACTORY was failing to load
+ implementation class when interface was loaded by a different
+ class loader. (Uwe Schindler, reported on java-user by Ahmed El-dawy)
New features
@@ -68,6 +114,12 @@
backwards compatibility. If Version < 3.1 is passed to the constructor,
LowerCaseFilter yields the old behavior. (Simon Willnauer, Robert Muir)
+* LUCENE-2034: Added ReusableAnalyzerBase, an abstract subclass of Analyzer
+ that makes it easier to reuse TokenStreams correctly. This issue also added
+ StopwordAnalyzerBase, which improves consistency of all Analyzers that use
+ stopwords, and implement many analyzers in contrib with it.
+ (Simon Willnauer via Robert Muir)
+
Optimizations
* LUCENE-2086: When resolving deleted terms, do so in term sort order
@@ -95,6 +147,33 @@
* LUCENE-2137: Switch to AtomicInteger for some ref counting (Earwin
Burrfoot via Mike McCandless)
+* LUCENE-2135: On IndexReader.close, forcefully evict any entries from
+ the FieldCache rather than waiting for the WeakHashMap to release
+ the reference (Mike McCandless)
+
+* LUCENE-2161: Improve concurrency of IndexReader, especially in the
+ context of near real-time readers. (Mike McCandless)
+
+* LUCENE-2164: ConcurrentMergeScheduler has more control over merge
+ threads. First, it gives smaller merges higher thread priority than
+ larges ones. Second, a new set/getMaxMergeCount setting will pause
+ the larger merges to allow smaller ones to finish. The defaults for
+ these settings are now dynamic, depending the number CPU cores as
+ reported by Runtime.getRuntime().availableProcessors() (Mike
+ McCandless)
+
+* LUCENE-2169: Improved CharArraySet.copy(), if source set is
+ also a CharArraySet. (Simon Willnauer via Uwe Schindler)
+
+* LUCENE-2084: Change IndexableBinaryStringTools to work on byte[] and char[]
+ directly, instead of Byte/CharBuffers, and modify CollationKeyFilter to
+ take advantage of this for faster performance.
+ (Steven Rowe, Uwe Schindler, Robert Muir)
+
+* LUCENE-2188: Add a utility class for tracking deprecated overridden
+ methods in non-final subclasses.
+ (Uwe Schindler, Robert Muir)
+
Build
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
@@ -116,6 +195,11 @@
index as well; improve javadocs of Filter to call out that the
provided reader is per-segment (Simon Willnauer via Mike McCandless)
+* LUCENE-2155: Fix time and zone dependent localization test failures
+ in queryparser tests. (Uwe Schindler, Chris Male, Robert Muir)
+
+* LUCENE-2170: Fix thread starvation problems. (Uwe Schindler)
+
======================= Release 3.0.0 2009-11-25 =======================
Changes in backwards compatibility policy
Modified: lucene/java/branches/flex_1458/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/build.xml?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/build.xml (original)
+++ lucene/java/branches/flex_1458/build.xml Thu Jan 14 19:05:12 2010
@@ -33,8 +33,8 @@
<pathelement location="${build.dir}/classes/java"/>
</path>
- <path id="tag.classpath">
- <pathelement location="${build.dir}/${tag}/classes/java"/>
+ <path id="backwards.classpath">
+ <pathelement location="${build.dir}/${backwards.branch}/classes/java"/>
</path>
<path id="demo.classpath">
@@ -76,72 +76,93 @@
<target name="test-core" depends="common.test"
description="Runs unit tests for the core Lucene code"
/>
- <target name="test" depends="test-core, test-contrib, test-tag"
+ <target name="test" depends="test-core, test-contrib, test-backwards"
description="Runs all unit tests (core, contrib and back-compat)"
/>
-
- <property name="tag" value="${compatibility.tag}"/>
- <path id="tag.test.classpath">
+ <path id="backwards.test.classpath">
<path refid="junit-path"/>
- <pathelement location="${build.dir}/${tag}/${tag}.jar"/>
+ <pathelement location="${build.dir}/${backwards.branch}/${backwards.branch}.jar"/>
</path>
- <path id="tag.junit.classpath">
+ <path id="backwards.junit.classpath">
<path refid="junit-path"/>
- <pathelement location="${build.dir}/${tag}/classes/test"/>
+ <pathelement location="${build.dir}/${backwards.branch}/classes/test"/>
<pathelement location="${build.dir}/${final.name}.jar"/>
<pathelement path="${java.class.path}"/>
</path>
- <target name="download-tag"
- description="Downloads a previous Lucene version. Specify tag version like this: -Dtag=lucene_2_4_back_compat_tests_20090127">
- <sequential>
- <mkdir dir="${tags.dir}/${tag}"/>
- <exec dir="${tags.dir}/${tag}" executable="${svn.exe}"
- failifexecutionfails="false">
- <arg line="checkout http://svn.apache.org/repos/asf/lucene/java/tags/${tag}/src/"/>
+ <available property="backwards.available" file="${backwards.dir}/${backwards.branch}/src"/>
+
+ <target name="init-backwards-checkout" unless="backwards.available" description="Creates the sparse svn checkout for backwards branch.">
+ <sequential>
+ <echo>Initial SVN checkout for '${backwards.branch}'...</echo>
+ <mkdir dir="${backwards.dir}"/>
+ <exec dir="${backwards.dir}" executable="${svn.exe}" failifexecutionfails="false" failonerror="true">
+ <arg line="checkout -r ${backwards.rev} --depth empty http://svn.apache.org/repos/asf/lucene/java/branches/${backwards.branch} ${backwards.branch}"/>
+ </exec>
+ <exec dir="${backwards.dir}" executable="${svn.exe}" failifexecutionfails="false" failonerror="true">
+ <arg line="update -r ${backwards.rev} --set-depth infinity ${backwards.branch}/src"/>
+ </exec>
+ </sequential>
+ </target>
+
+ <target name="download-backwards" depends="init-backwards-checkout" description="Downloads a previous Lucene version.">
+ <sequential>
+ <echo>Update backwards branch '${backwards.branch}' to revision ${backwards.rev}...</echo>
+ <exec dir="${backwards.dir}" executable="${svn.exe}" failifexecutionfails="false" failonerror="true">
+ <arg line="update -r ${backwards.rev} ${backwards.branch}"/>
</exec>
- </sequential>
+ </sequential>
</target>
- <target name="test-tag" depends="download-tag, compile-core, jar-core"
- description="Runs tests of a previous Lucene version. Specify tag version like this: -Dtag=branches/lucene_2_9_back_compat_tests">
+ <!-- remove this -->
+ <target name="test-tag" depends="test-backwards" description="deprecated"/>
+
+ <target name="test-backwards" depends="download-backwards, compile-core, jar-core"
+ description="Runs tests of a previous Lucene version.">
<sequential>
- <available property="tag.available" file="${tags.dir}/${tag}/src/test" />
+ <available property="backwards.tests.available" file="${backwards.dir}/${backwards.branch}/src/test" />
- <fail unless="tag.available">
- Specified tag '${tag}' could not be found in directory '${tags.dir}/${tag}'.
+ <fail unless="backwards.tests.available">
+ Backwards branch '${backwards.branch}' could not be found in directory '${backwards.dir}'.
</fail>
- <mkdir dir="${build.dir}/${tag}"/>
+ <mkdir dir="${build.dir}/${backwards.branch}"/>
- <!-- first compile tag classes -->
+ <!-- first compile branch classes -->
<compile
- srcdir="${tags.dir}/${tag}/src/java"
- destdir="${build.dir}/${tag}/classes/java"
- javac.source="${javac.source.tag}" javac.target="${javac.target.tag}">
- <classpath refid="tag.classpath"/>
+ srcdir="${backwards.dir}/${backwards.branch}/src/java"
+ destdir="${build.dir}/${backwards.branch}/classes/java"
+ javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}">
+ <classpath refid="backwards.classpath"/>
</compile>
- <!-- compile jar for tag -->
- <jarify basedir="${build.dir}/${tag}/classes/java"
- destfile="${build.dir}/${tag}/${tag}.jar" />
+ <!-- compile jar for branch -->
+ <jarify basedir="${build.dir}/${backwards.branch}/classes/java"
+ destfile="${build.dir}/${backwards.branch}/${backwards.branch}.jar" />
- <!-- compile tag tests against tag jar -->
- <compile-test-macro srcdir="${tags.dir}/${tag}/src/test" destdir="${build.dir}/${tag}/classes/test"
- test.classpath="tag.test.classpath" javac.source="${javac.source.tag}" javac.target="${javac.target.tag}"/>
+ <!-- compile branch tests against branch jar -->
+ <compile-test-macro srcdir="${backwards.dir}/${backwards.branch}/src/test" destdir="${build.dir}/${backwards.branch}/classes/test"
+ test.classpath="backwards.test.classpath" javac.source="${javac.source.backwards}" javac.target="${javac.target.backwards}"/>
- <!-- run tag tests against trunk jar -->
- <test-macro dataDir="${tags.dir}/${tag}/src/test"
- tempDir="${build.dir}/${tag}"
- junit.classpath="tag.junit.classpath"/>
+ <!-- run branch tests against trunk jar -->
+ <test-macro dataDir="${backwards.dir}/${backwards.branch}/src/test"
+ tempDir="${build.dir}/${backwards.branch}"
+ junit.classpath="backwards.junit.classpath"
+ junit.output.dir="${junit.output.dir.backwards}" />
</sequential>
</target>
+ <!-- remove this soon -->
<target name="clean-tags"
- description="Removes contents of the tags directory">
- <delete dir="${tags.dir}"/>
+ description="deprecated: removes the old 'tag' subdir">
+ <delete dir="${common.dir}/tags"/>
+ </target>
+
+ <target name="clean-backwards"
+ description="Removes contents of the '${backwards.dir}' directory">
+ <delete dir="${backwards.dir}"/>
</target>
<!-- ================================================================== -->
Modified: lucene/java/branches/flex_1458/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/common-build.xml?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/common-build.xml (original)
+++ lucene/java/branches/flex_1458/common-build.xml Thu Jan 14 19:05:12 2010
@@ -42,7 +42,8 @@
<property name="Name" value="Lucene"/>
<property name="dev.version" value="3.1-dev"/>
<property name="version" value="${dev.version}"/>
- <property name="compatibility.tag" value="flex_1458_3_0_back_compat_tests_20091211a"/>
+ <property name="backwards.branch" value="flex_1458_3_0_back_compat_tests"/>
+ <property name="backwards.rev" value="899354"/>
<property name="spec.version" value="${version}"/>
<property name="year" value="2000-${current.year}"/>
<property name="final.name" value="lucene-${name}-${version}"/>
@@ -60,8 +61,8 @@
<property name="javac.debug" value="on"/>
<property name="javac.source" value="1.5"/>
<property name="javac.target" value="1.5"/>
- <property name="javac.source.tag" value="1.5"/>
- <property name="javac.target.tag" value="1.5"/>
+ <property name="javac.source.backwards" value="1.5"/>
+ <property name="javac.target.backwards" value="1.5"/>
<property name="javadoc.link" value="http://java.sun.com/j2se/1.5/docs/api/"/>
<property name="javadoc.access" value="protected"/>
@@ -85,7 +86,7 @@
<property name="src.dir" location="src/java"/>
<property name="build.dir" location="build"/>
- <property name="tags.dir" location="tags"/>
+ <property name="backwards.dir" location="backwards"/>
<property name="dist.dir" location="dist"/>
<property name="maven.dist.dir" location="dist/maven"/>
<property name="m2.repository.url" value="file://${maven.dist.dir}"/>
@@ -95,7 +96,9 @@
<property name="jflex.home" location="${common.dir}"/>
<property name="junit.output.dir" location="${build.dir}/test"/>
+ <property name="junit.output.dir.backwards" location="${build.dir}/bw-test"/>
<property name="junit.reports" location="${build.dir}/test/reports"/>
+ <property name="junit.reports.backwards" location="${build.dir}/bw-test/reports"/>
<property name="junit.includes" value="**/Test*.java,**/*Test.java"/>
<property name="junit.excludes" value=""/>
@@ -128,7 +131,6 @@
</and>
</condition>
-
<available
property="javacc.present"
classname="org.javacc.parser.Main"
@@ -464,21 +466,18 @@
See http://issues.apache.org/jira/browse/LUCENE-721
-->
- <target name="clover" depends="clover.setup, clover.info" description="Instrument the Unit tests using Clover. Requires a Clover 1.3.2 license and clover.jar in the ANT classpath. To use, specify -Drun.clover=true on the command line."/>
+ <target name="clover" depends="clover.setup, clover.info" description="Instrument the Unit tests using Clover. Requires a Clover 2.x license and clover.jar in the ANT classpath. To use, specify -Drun.clover=true on the command line."/>
<target name="clover.setup" if="clover.enabled">
- <taskdef resource="clovertasks"/>
+ <taskdef resource="cloverlib.xml"/>
<mkdir dir="${clover.db.dir}"/>
- <clover-setup initString="${clover.db.dir}/lucene_coverage.db">
+ <clover-setup initString="${clover.db.dir}/lucene_coverage.db" encoding="${build.encoding}">
<fileset dir="src/java">
- <!-- see https://svn.apache.org/repos/private/committers/donated-licenses/clover/1.3.2/README.txt -->
- <include name="org/apache/**" />
- <include name="javax/**" />
- <include name="org/xml/**" />
- <include name="org/w3c/**" />
- <include name="com/example/**" />
- <exclude name="org/apache/lucene/analysis/ASCIIFoldingFilter.java" /> <!-- Class too large for clover -->
+ <include name="org/apache/**/*.java" />
</fileset>
+ <testsources dir="src/test">
+ <include name="org/apache/**/*.java" />
+ </testsources>
</clover-setup>
</target>
@@ -502,12 +501,29 @@
-->
<target name="generate-clover-reports" depends="clover.check, clover">
<mkdir dir="${clover.report.dir}"/>
+ <!-- This extra path is needed, because from a top-level ant run, the contrib tests would be not visible (report generation is only done on top-level not via subants) -->
+ <fileset dir="contrib" id="clover.contrib.test.src.files">
+ <include name="**/test/**/*.java"/>
+ </fileset>
+ <fileset dir="src/test" id="clover.test.src.files">
+ <include name="**/*.java" />
+ </fileset>
+ <fileset dir="${build.dir}" id="clover.test.result.files">
+ <include name="**/test/TEST-*.xml" />
+ <!-- do not include BW tests -->
+ </fileset>
<clover-report>
- <current outfile="${clover.report.dir}" title="${final.name}">
- <format type="html"/>
+ <current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
+ <format type="html" filter="assert"/>
+ <testsources refid="clover.test.src.files"/>
+ <testsources refid="clover.contrib.test.src.files"/>
+ <testresults refid="clover.test.result.files"/>
</current>
<current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
- <format type="xml"/>
+ <format type="xml" filter="assert"/>
+ <testsources refid="clover.test.src.files"/>
+ <testsources refid="clover.contrib.test.src.files"/>
+ <testresults refid="clover.test.result.files"/>
</current>
</clover-report>
</target>
@@ -523,6 +539,17 @@
</fileset>
<report format="frames" todir="${junit.reports}"/>
</junitreport>
+
+ <mkdir dir="${junit.reports.backwards}"/>
+ <junitreport todir="${junit.output.dir.backwards}">
+ <!-- this fileset let's the task work for individual contribs,
+ as well as the project as a whole
+ -->
+ <fileset dir="${build.dir}">
+ <include name="**/bw-test/TEST-*.xml"/>
+ </fileset>
+ <report format="frames" todir="${junit.reports.backwards}"/>
+ </junitreport>
</target>
<target name="jar" depends="jar-core">
Propchange: lucene/java/branches/flex_1458/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 14 19:05:12 2010
@@ -1,5 +1,5 @@
/lucene/java/branches/lucene_2_4/contrib:748824
-/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036
+/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
-/lucene/java/branches/lucene_3_0/contrib:880793
-/lucene/java/trunk/contrib:824912-825292,827043-833960,880727-886190,889185,889622,889667
+/lucene/java/branches/lucene_3_0/contrib:880793,896906
+/lucene/java/trunk/contrib:824912-825292,827043-833960,880727-886190,889185,889622,889667,889866-899001
Modified: lucene/java/branches/flex_1458/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/CHANGES.txt?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/CHANGES.txt (original)
+++ lucene/java/branches/flex_1458/contrib/CHANGES.txt Thu Jan 14 19:05:12 2010
@@ -9,6 +9,13 @@
compose their own analyzers instead of subclassing existing ones.
(Simon Willnauer)
+ * LUCENE-2194, LUCENE-2201: Snowball APIs were upgraded to snowball revision
+ 502 (with some local modifications for improved performance).
+ Index backwards compatibility and binary backwards compatibility is
+ preserved, but some protected/public member variables changed type. This
+ does NOT affect java code/class files produced by the snowball compiler,
+ but technically is a backwards compatibility break. (Robert Muir)
+
Changes in runtime behavior
* LUCENE-2117: SnowballAnalyzer uses TurkishLowerCaseFilter instead of
@@ -18,6 +25,9 @@
Bug fixes
+ * LUCENE-2199: ShingleFilter skipped over tri-gram shingles if outputUnigram
+ was set to false. (Simon Willnauer)
+
* LUCENE-2068: Fixed ReverseStringFilter which was not aware of supplementary
characters. During reverse the filter created unpaired surrogates, which
will be replaced by U+FFFD by the indexer, but not at query time. The filter
@@ -28,11 +38,24 @@
correctly (enumerate all non-deleted docs). (Karl Wettin via Mike
McCandless)
+ * LUCENE-2035: TokenSources.getTokenStream() does not assign positionIncrement.
+ (Christopher Morris via Mark Miller)
+
API Changes
* LUCENE-2108: Add SpellChecker.close, to close the underlying
reader. (Eirik Bjørsnøs via Mike McCandless)
+
+ * LUCENE-2147: Spatial GeoHashUtils now always decode GeoHash strings
+ with full precision. GeoHash#decode_exactly(String) was merged into
+ GeoHash#decode(String). (Chris Male, Simon Willnauer)
+
+ * LUCENE-2165: Add a constructor to SnowballAnalyzer that takes a Set of
+ stopwords, and deprecate the String[] one. (Nick Burch via Robert Muir)
+ * LUCENE-2204: Change some package private classes/members to publicly accessible to implement
+ custom FragmentsBuilders. (Koji Sekiguchi)
+
New features
* LUCENE-2102: Add a Turkish LowerCase Filter. TurkishLowerCaseFilter handles
@@ -63,7 +86,18 @@
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
into core, and moved the ICU-based collation support into contrib/icu.
- (Robert Muir)
+ (Steven Rowe, Robert Muir)
+
+Optimizations
+
+ * LUCENE-2157: DelimitedPayloadTokenFilter no longer copies the buffer
+ over itsself. Instead it sets only the length. This patch also optimizes
+ the logic of the filter and uses NIO for IdentityEncoder. (Uwe Schindler)
+
+ * LUCENE-2084: Change IndexableBinaryStringTools to work on byte[] and char[]
+ directly, instead of Byte/CharBuffers, and modify ICUCollationKeyFilter to
+ take advantage of this for faster performance.
+ (Steven Rowe, Uwe Schindler, Robert Muir)
Test Cases
Propchange: lucene/java/branches/flex_1458/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 14 19:05:12 2010
@@ -1,5 +1,5 @@
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036
+/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
-/lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793
-/lucene/java/trunk/contrib/CHANGES.txt:829439-833960,880727-886190,889185,889622,889667
+/lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793,896906
+/lucene/java/trunk/contrib/CHANGES.txt:829439-833960,880727-886190,889185,889622,889667,889866-899001
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -19,17 +19,15 @@
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.Reader;
-import java.util.Collections;
import java.util.Hashtable;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
@@ -52,7 +50,7 @@
* </ul>
*
*/
-public final class ArabicAnalyzer extends Analyzer {
+public final class ArabicAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default Arabic stopwords.
@@ -63,20 +61,18 @@
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
/**
- * Contains the stopwords used with the StopFilter.
- */
- private final Set<?> stoptable;
- /**
* The comment character in the stopwords file. All lines prefixed with this will be ignored
* @deprecated use {@link WordlistLoader#getWordSet(File, String)} directly
*/
+ // TODO make this private
+ @Deprecated
public static final String STOPWORDS_COMMENT = "#";
/**
* Returns an unmodifiable instance of the default stop-words set.
* @return an unmodifiable instance of the default stop-words set.
*/
- public static Set<String> getDefaultStopSet(){
+ public static Set<?> getDefaultStopSet(){
return DefaultSetHolder.DEFAULT_STOP_SET;
}
@@ -85,34 +81,19 @@
* accesses the static final set the first time.;
*/
private static class DefaultSetHolder {
- static final Set<String> DEFAULT_STOP_SET;
+ static final Set<?> DEFAULT_STOP_SET;
static {
try {
- DEFAULT_STOP_SET = loadDefaultStopWordSet();
+ DEFAULT_STOP_SET = loadStopwordSet(false, ArabicAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
throw new RuntimeException("Unable to load default stopword set");
}
}
-
- static Set<String> loadDefaultStopWordSet() throws IOException {
- InputStream stream = ArabicAnalyzer.class
- .getResourceAsStream(DEFAULT_STOPWORD_FILE);
- try {
- InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
- // make sure it is unmodifiable as we expose it in the outer class
- return Collections.unmodifiableSet(WordlistLoader.getWordSet(reader,
- STOPWORDS_COMMENT));
- } finally {
- stream.close();
- }
- }
}
- private final Version matchVersion;
-
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
*/
@@ -129,14 +110,14 @@
* a stopword set
*/
public ArabicAnalyzer(Version matchVersion, Set<?> stopwords){
- stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stopwords));
- this.matchVersion = matchVersion;
+ super(matchVersion, stopwords);
}
/**
* Builds an analyzer with the given stop words.
* @deprecated use {@link #ArabicAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public ArabicAnalyzer( Version matchVersion, String... stopwords ) {
this(matchVersion, StopFilter.makeStopSet(matchVersion, stopwords ));
}
@@ -145,6 +126,7 @@
* Builds an analyzer with the given stop words.
* @deprecated use {@link #ArabicAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public ArabicAnalyzer( Version matchVersion, Hashtable<?,?> stopwords ) {
this(matchVersion, stopwords.keySet());
}
@@ -153,60 +135,28 @@
* Builds an analyzer with the given stop words. Lines can be commented out using {@link #STOPWORDS_COMMENT}
* @deprecated use {@link #ArabicAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public ArabicAnalyzer( Version matchVersion, File stopwords ) throws IOException {
this(matchVersion, WordlistLoader.getWordSet( stopwords, STOPWORDS_COMMENT));
}
/**
- * Creates a {@link TokenStream} which tokenizes all the text in the provided {@link Reader}.
+ * Creates {@link TokenStreamComponents} used to tokenize all the text in the provided {@link Reader}.
*
- * @return A {@link TokenStream} built from an {@link ArabicLetterTokenizer} filtered with
+ * @return {@link TokenStreamComponents} built from an {@link ArabicLetterTokenizer} filtered with
* {@link LowerCaseFilter}, {@link StopFilter}, {@link ArabicNormalizationFilter}
* and {@link ArabicStemFilter}.
*/
@Override
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new ArabicLetterTokenizer( reader );
- result = new LowerCaseFilter(matchVersion, result);
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ final Tokenizer source = new ArabicLetterTokenizer(reader);
+ TokenStream result = new LowerCaseFilter(matchVersion, source);
// the order here is important: the stopword list is not normalized!
- result = new StopFilter( matchVersion, result, stoptable );
- result = new ArabicNormalizationFilter( result );
- result = new ArabicStemFilter( result );
-
- return result;
- }
-
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
-
- /**
- * Returns a (possibly reused) {@link TokenStream} which tokenizes all the text
- * in the provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from an {@link ArabicLetterTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link StopFilter}, {@link ArabicNormalizationFilter}
- * and {@link ArabicStemFilter}.
- */
- @Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader)
- throws IOException {
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new ArabicLetterTokenizer(reader);
- streams.result = new LowerCaseFilter(matchVersion, streams.source);
- // the order here is important: the stopword list is not normalized!
- streams.result = new StopFilter( matchVersion, streams.result, stoptable);
- streams.result = new ArabicNormalizationFilter(streams.result);
- streams.result = new ArabicStemFilter(streams.result);
- setPreviousTokenStream(streams);
- } else {
- streams.source.reset(reader);
- }
- return streams.result;
+ result = new StopFilter( matchVersion, result, stopwords);
+ result = new ArabicNormalizationFilter(result);
+ return new TokenStreamComponents(source, new ArabicStemFilter(result));
}
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java Thu Jan 14 19:05:12 2010
@@ -30,8 +30,8 @@
public final class ArabicNormalizationFilter extends TokenFilter {
- protected ArabicNormalizer normalizer = null;
- private TermAttribute termAtt;
+ private final ArabicNormalizer normalizer;
+ private final TermAttribute termAtt;
public ArabicNormalizationFilter(TokenStream input) {
super(input);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -17,17 +17,16 @@
* limitations under the License.
*/
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.Reader;
-import java.util.Collections;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
@@ -43,7 +42,7 @@
* http://members.unine.ch/jacques.savoy/Papers/BUIR.pdf
* <p>
*/
-public final class BulgarianAnalyzer extends Analyzer {
+public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
/**
* File containing default Bulgarian stopwords.
@@ -55,13 +54,12 @@
public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";
/**
- * Contains the stopwords used with the StopFilter.
- */
- private final Set<?> stoptable;
- /**
* The comment character in the stopwords file. All lines prefixed with this
* will be ignored
+ * @deprecated use {@link WordlistLoader#getWordSet(File, String)} directly
*/
+ //TODO make this private
+ @Deprecated
public static final String STOPWORDS_COMMENT = "#";
/**
@@ -69,7 +67,7 @@
*
* @return an unmodifiable instance of the default stop-words set.
*/
- public static Set<String> getDefaultStopSet() {
+ public static Set<?> getDefaultStopSet() {
return DefaultSetHolder.DEFAULT_STOP_SET;
}
@@ -78,35 +76,19 @@
* class accesses the static final set the first time.;
*/
private static class DefaultSetHolder {
- static final Set<String> DEFAULT_STOP_SET;
+ static final Set<?> DEFAULT_STOP_SET;
static {
try {
- DEFAULT_STOP_SET = loadDefaultStopWordSet();
- } catch (Exception ex) {
+ DEFAULT_STOP_SET = loadStopwordSet(false, BulgarianAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
+ } catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)
- throw new RuntimeException("Unable to load default stopword set", ex);
- }
- }
-
- static Set<String> loadDefaultStopWordSet() throws IOException {
- final InputStream stream = BulgarianAnalyzer.class
- .getResourceAsStream(DEFAULT_STOPWORD_FILE);
- try {
- InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
- // make sure it is unmodifiable as we expose it in the outer class
- return Collections.unmodifiableSet(WordlistLoader.getWordSet(reader,
- STOPWORDS_COMMENT));
- } finally {
- if(stream != null)
- stream.close();
+ throw new RuntimeException("Unable to load default stopword set");
}
}
}
-
- private final Version matchVersion;
-
+
/**
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
@@ -119,58 +101,24 @@
* Builds an analyzer with the given stop words.
*/
public BulgarianAnalyzer(Version matchVersion, Set<?> stopwords) {
- super();
- stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion,
- stopwords));
- this.matchVersion = matchVersion;
+ super(matchVersion, stopwords);
}
/**
- * Creates a {@link TokenStream} which tokenizes all the text in the provided
+ * Creates a {@link TokenStreamComponents} which tokenizes all the text in the provided
* {@link Reader}.
*
- * @return A {@link TokenStream} built from an {@link StandardTokenizer}
+ * @return A {@link TokenStreamComponents} built from an {@link StandardTokenizer}
* filtered with {@link StandardFilter}, {@link LowerCaseFilter},
* {@link StopFilter}, and {@link BulgarianStemFilter}.
*/
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer(matchVersion, reader);
- result = new StandardFilter(result);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ TokenStream result = new StandardFilter(source);
result = new LowerCaseFilter(matchVersion, result);
- result = new StopFilter(matchVersion, result, stoptable);
+ result = new StopFilter(matchVersion, result, stopwords);
result = new BulgarianStemFilter(result);
- return result;
- }
-
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
-
- /**
- * Returns a (possibly reused) {@link TokenStream} which tokenizes all the
- * text in the provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from an {@link StandardTokenizer}
- * filtered with {@link StandardFilter}, {@link LowerCaseFilter},
- * {@link StopFilter}, and {@link BulgarianStemFilter}.
- */
- @Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader)
- throws IOException {
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new StandardTokenizer(matchVersion, reader);
- streams.result = new StandardFilter(streams.source);
- streams.result = new LowerCaseFilter(matchVersion, streams.result);
- streams.result = new StopFilter(matchVersion, streams.result, stoptable);
- streams.result = new BulgarianStemFilter(streams.result);
- setPreviousTokenStream(streams);
- } else {
- streams.source.reset(reader);
- }
- return streams.result;
+ return new TokenStreamComponents(source, result);
}
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -21,19 +21,21 @@
import java.io.IOException;
import java.io.Reader;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import java.util.Collections;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
-import org.apache.lucene.analysis.standard.StandardAnalyzer; // for javadoc
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.Version;
@@ -49,13 +51,14 @@
* <p><b>NOTE</b>: This class uses the same {@link Version}
* dependent settings as {@link StandardAnalyzer}.</p>
*/
-public final class BrazilianAnalyzer extends Analyzer {
+public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
/**
* List of typical Brazilian Portuguese stopwords.
* @deprecated use {@link #getDefaultStopSet()} instead
*/
// TODO make this private in 3.1
+ @Deprecated
public final static String[] BRAZILIAN_STOP_WORDS = {
"a","ainda","alem","ambas","ambos","antes",
"ao","aonde","aos","apos","aquele","aqueles",
@@ -91,19 +94,13 @@
Arrays.asList(BRAZILIAN_STOP_WORDS), false));
}
- /**
- * Contains the stopwords used with the {@link StopFilter}.
- */
- private final Set<?> stoptable;
-
+
/**
* Contains words that should be indexed but not stemmed.
*/
// TODO make this private in 3.1
private Set<?> excltable = Collections.emptySet();
- private final Version matchVersion;
-
/**
* Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
*/
@@ -120,8 +117,7 @@
* a stopword set
*/
public BrazilianAnalyzer(Version matchVersion, Set<?> stopwords) {
- stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stopwords));
- this.matchVersion = matchVersion;
+ super(matchVersion, stopwords);
}
/**
@@ -143,6 +139,7 @@
* Builds an analyzer with the given stop words.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public BrazilianAnalyzer(Version matchVersion, String... stopwords) {
this(matchVersion, StopFilter.makeStopSet(matchVersion, stopwords));
}
@@ -151,6 +148,7 @@
* Builds an analyzer with the given stop words.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public BrazilianAnalyzer(Version matchVersion, Map<?,?> stopwords) {
this(matchVersion, stopwords.keySet());
}
@@ -159,6 +157,7 @@
* Builds an analyzer with the given stop words.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public BrazilianAnalyzer(Version matchVersion, File stopwords)
throws IOException {
this(matchVersion, WordlistLoader.getWordSet(stopwords));
@@ -168,6 +167,7 @@
* Builds an exclusionlist from an array of Strings.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set, Set)} instead
*/
+ @Deprecated
public void setStemExclusionTable( String... exclusionlist ) {
excltable = StopFilter.makeStopSet( matchVersion, exclusionlist );
setPreviousTokenStream(null); // force a new stemmer to be created
@@ -176,6 +176,7 @@
* Builds an exclusionlist from a {@link Map}.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set, Set)} instead
*/
+ @Deprecated
public void setStemExclusionTable( Map<?,?> exclusionlist ) {
excltable = new HashSet<Object>(exclusionlist.keySet());
setPreviousTokenStream(null); // force a new stemmer to be created
@@ -184,57 +185,27 @@
* Builds an exclusionlist from the words contained in the given file.
* @deprecated use {@link #BrazilianAnalyzer(Version, Set, Set)} instead
*/
+ @Deprecated
public void setStemExclusionTable( File exclusionlist ) throws IOException {
excltable = WordlistLoader.getWordSet( exclusionlist );
setPreviousTokenStream(null); // force a new stemmer to be created
}
-
- /**
- * Creates a {@link TokenStream} which tokenizes all the text in the provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link StandardFilter}, {@link StopFilter}, and
- * {@link BrazilianStemFilter}.
- */
- @Override
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer( matchVersion, reader );
- result = new LowerCaseFilter( matchVersion, result );
- result = new StandardFilter( result );
- result = new StopFilter( matchVersion, result, stoptable );
- result = new BrazilianStemFilter( result, excltable );
- return result;
- }
-
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
-
- /**
- * Returns a (possibly reused) {@link TokenStream} which tokenizes all the text
- * in the provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
- * {@link LowerCaseFilter}, {@link StandardFilter}, {@link StopFilter}, and
- * {@link BrazilianStemFilter}.
- */
- @Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader)
- throws IOException {
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new StandardTokenizer(matchVersion, reader);
- streams.result = new LowerCaseFilter(matchVersion, streams.source);
- streams.result = new StandardFilter(streams.result);
- streams.result = new StopFilter(matchVersion, streams.result, stoptable);
- streams.result = new BrazilianStemFilter(streams.result, excltable);
- setPreviousTokenStream(streams);
- } else {
- streams.source.reset(reader);
- }
- return streams.result;
- }
+ /**
+ * Creates {@link TokenStreamComponents} used to tokenize all the text in the provided {@link Reader}.
+ *
+ * @return {@link TokenStreamComponents} built from a {@link StandardTokenizer} filtered with
+ * {@link LowerCaseFilter}, {@link StandardFilter}, {@link StopFilter}, and
+ * {@link BrazilianStemFilter}.
+ */
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ TokenStream result = new LowerCaseFilter(matchVersion, source);
+ result = new StandardFilter(result);
+ result = new StopFilter(matchVersion, result, stopwords);
+ return new TokenStreamComponents(source, new BrazilianStemFilter(result,
+ excltable));
+ }
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -19,12 +19,12 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.analysis.StopFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util.Version;
-import java.io.IOException;
import java.io.Reader;
import java.util.Arrays;
import java.util.Set;
@@ -35,7 +35,7 @@
* filters with {@link StopFilter}
*
*/
-public final class CJKAnalyzer extends Analyzer {
+public final class CJKAnalyzer extends StopwordAnalyzerBase {
//~ Static fields/initializers ---------------------------------------------
/**
@@ -45,6 +45,7 @@
*/
// TODO make this final in 3.1 -
// this might be revised and merged with StopFilter stop words too
+ @Deprecated
public final static String[] STOP_WORDS = {
"a", "and", "are", "as", "at", "be",
"but", "by", "for", "if", "in",
@@ -71,11 +72,6 @@
.unmodifiableSet(new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(STOP_WORDS),
false));
}
- /**
- * stop word list
- */
- private final Set<?> stopTable;
- private final Version matchVersion;
//~ Constructors -----------------------------------------------------------
@@ -95,8 +91,7 @@
* a stopword set
*/
public CJKAnalyzer(Version matchVersion, Set<?> stopwords){
- stopTable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stopwords));
- this.matchVersion = matchVersion;
+ super(matchVersion, stopwords);
}
/**
@@ -105,52 +100,17 @@
* @param stopWords stop word array
* @deprecated use {@link #CJKAnalyzer(Version, Set)} instead
*/
+ @Deprecated
public CJKAnalyzer(Version matchVersion, String... stopWords) {
- stopTable = StopFilter.makeStopSet(matchVersion, stopWords);
- this.matchVersion = matchVersion;
+ super(matchVersion, StopFilter.makeStopSet(matchVersion, stopWords));
}
//~ Methods ----------------------------------------------------------------
- /**
- * Creates a {@link TokenStream} which tokenizes all the text in the provided {@link Reader}.
- *
- * @param fieldName lucene field name
- * @param reader input {@link Reader}
- * @return A {@link TokenStream} built from {@link CJKTokenizer}, filtered with
- * {@link StopFilter}
- */
- @Override
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new StopFilter(matchVersion, new CJKTokenizer(reader), stopTable);
- }
-
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
-
- /**
- * Returns a (possibly reused) {@link TokenStream} which tokenizes all the text
- * in the provided {@link Reader}.
- *
- * @param fieldName lucene field name
- * @param reader Input {@link Reader}
- * @return A {@link TokenStream} built from {@link CJKTokenizer}, filtered with
- * {@link StopFilter}
- */
@Override
- public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- /* tokenStream() is final, no back compat issue */
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new CJKTokenizer(reader);
- streams.result = new StopFilter(matchVersion, streams.source, stopTable);
- setPreviousTokenStream(streams);
- } else {
- streams.source.reset(reader);
- }
- return streams.result;
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ final Tokenizer source = new CJKTokenizer(reader);
+ return new TokenStreamComponents(source, new StopFilter(matchVersion, source, stopwords));
}
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java Thu Jan 14 19:05:12 2010
@@ -25,8 +25,6 @@
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.AttributeSource.AttributeFactory;
-
/**
* CJKTokenizer is designed for Chinese, Japanese, and Korean languages.
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -17,10 +17,11 @@
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Reader;
+
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
/**
@@ -29,49 +30,19 @@
*
*/
-public final class ChineseAnalyzer extends Analyzer {
-
- public ChineseAnalyzer() {
- }
-
- /**
- * Creates a {@link TokenStream} which tokenizes all the text in the provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from a {@link ChineseTokenizer}
- * filtered with {@link ChineseFilter}.
- */
- @Override
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new ChineseTokenizer(reader);
- result = new ChineseFilter(result);
- return result;
- }
-
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
+public final class ChineseAnalyzer extends ReusableAnalyzerBase {
- /**
- * Returns a (possibly reused) {@link TokenStream} which tokenizes all the text in the
- * provided {@link Reader}.
- *
- * @return A {@link TokenStream} built from a {@link ChineseTokenizer}
- * filtered with {@link ChineseFilter}.
- */
+ /**
+ * Creates {@link TokenStreamComponents} used to tokenize all the text in the
+ * provided {@link Reader}.
+ *
+ * @return {@link TokenStreamComponents} built from a
+ * {@link ChineseTokenizer} filtered with {@link ChineseFilter}
+ */
@Override
- public final TokenStream reusableTokenStream(String fieldName, Reader reader)
- throws IOException {
- /* tokenStream() is final, no back compat issue */
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new ChineseTokenizer(reader);
- streams.result = new ChineseFilter(streams.source);
- setPreviousTokenStream(streams);
- } else {
- streams.source.reset(reader);
- }
- return streams.result;
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ final Tokenizer source = new ChineseTokenizer(reader);
+ return new TokenStreamComponents(source, new ChineseFilter(source));
}
}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java Thu Jan 14 19:05:12 2010
@@ -80,6 +80,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, String[], int, int, int, boolean)} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, String[] dictionary, int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
this(Version.LUCENE_30, input, makeDictionary(dictionary),minWordSize,minSubwordSize,maxSubwordSize, onlyLongestMatch);
}
@@ -87,6 +88,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, String[], boolean)} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, String[] dictionary, boolean onlyLongestMatch) {
this(Version.LUCENE_30, input, makeDictionary(dictionary),DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, onlyLongestMatch);
}
@@ -94,6 +96,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, Set, boolean)} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, Set<?> dictionary, boolean onlyLongestMatch) {
this(Version.LUCENE_30, input, dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, onlyLongestMatch);
}
@@ -101,6 +104,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, String[])} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, String[] dictionary) {
this(Version.LUCENE_30, input, makeDictionary(dictionary),DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, false);
}
@@ -108,6 +112,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, Set)} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, Set<?> dictionary) {
this(Version.LUCENE_30, input, dictionary,DEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE, false);
}
@@ -115,6 +120,7 @@
/**
* @deprecated use {@link #CompoundWordTokenFilterBase(Version, TokenStream, Set, int, int, int, boolean)} instead
*/
+ @Deprecated
protected CompoundWordTokenFilterBase(TokenStream input, Set<?> dictionary, int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
this(Version.LUCENE_30, input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
}