You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/01/25 22:56:51 UTC
svn commit: r1235969 [1/9] - in /lucene/dev/branches/lucene2858: ./
dev-tools/eclipse/ dev-tools/idea/lucene/contrib/ dev-tools/maven/ lucene/
lucene/contrib/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/
lucene/src/java/org/...
Author: uschindler
Date: Wed Jan 25 21:56:44 2012
New Revision: 1235969
URL: http://svn.apache.org/viewvc?rev=1235969&view=rev
Log:
LUCENE-2858: Reverse merged revision(s) 1-0 from lucene/dev/trunk
Added:
lucene/dev/branches/lucene2858/solr/cloud-dev/ (props changed)
- copied from r1235968, lucene/dev/trunk/solr/cloud-dev/
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/commons-compress-1.3.jar
- copied unchanged from r1235968, lucene/dev/trunk/solr/contrib/extraction/lib/commons-compress-1.3.jar
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/tika-core-1.0.jar
- copied unchanged from r1235968, lucene/dev/trunk/solr/contrib/extraction/lib/tika-core-1.0.jar
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/tika-parsers-1.0.jar
- copied unchanged from r1235968, lucene/dev/trunk/solr/contrib/extraction/lib/tika-parsers-1.0.jar
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/Overseer.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/ShardLeaderWatcher.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ShardLeaderWatcher.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/PeerSync.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/PeerSync.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/TransactionLog.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/TransactionLog.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/VersionBucket.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/VersionBucket.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/VersionInfo.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/VersionInfo.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/search/TestRecovery.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestRecovery.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
lucene/dev/branches/lucene2858/solr/lib/apache-solr-noggit-r1211150.jar
- copied unchanged from r1235968, lucene/dev/trunk/solr/lib/apache-solr-noggit-r1211150.jar
lucene/dev/branches/lucene2858/solr/lib/zookeeper-3.3.4.jar
- copied unchanged from r1235968, lucene/dev/trunk/solr/lib/zookeeper-3.3.4.jar
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequestExt.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequestExt.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkOperation.java
- copied unchanged from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/ZkOperation.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/zookeeper/
- copied from r1235968, lucene/dev/trunk/solr/solrj/src/java/org/apache/zookeeper/
Removed:
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/commons-compress-1.2.jar
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/tika-core-0.10.jar
lucene/dev/branches/lucene2858/solr/contrib/extraction/lib/tika-parsers-0.10.jar
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/FSUpdateLog.java
lucene/dev/branches/lucene2858/solr/lib/apache-solr-noggit-r1209632.jar
lucene/dev/branches/lucene2858/solr/lib/zookeeper-3.3.3.jar
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java
Modified:
lucene/dev/branches/lucene2858/ (props changed)
lucene/dev/branches/lucene2858/dev-tools/eclipse/dot.classpath
lucene/dev/branches/lucene2858/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/lucene2858/dev-tools/maven/pom.xml.template
lucene/dev/branches/lucene2858/lucene/ (props changed)
lucene/dev/branches/lucene2858/lucene/CHANGES.txt
lucene/dev/branches/lucene2858/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/branches/lucene2858/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (props changed)
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/TermStatistics.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java
lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
lucene/dev/branches/lucene2858/modules/analysis/kuromoji/ (props changed)
lucene/dev/branches/lucene2858/modules/benchmark/ (props changed)
lucene/dev/branches/lucene2858/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java
lucene/dev/branches/lucene2858/modules/facet/ (props changed)
lucene/dev/branches/lucene2858/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/lucene2858/solr/ (props changed)
lucene/dev/branches/lucene2858/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene2858/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene2858/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene2858/solr/README.txt (props changed)
lucene/dev/branches/lucene2858/solr/build.xml (contents, props changed)
lucene/dev/branches/lucene2858/solr/client/ (props changed)
lucene/dev/branches/lucene2858/solr/common-build.xml (props changed)
lucene/dev/branches/lucene2858/solr/contrib/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
lucene/dev/branches/lucene2858/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
lucene/dev/branches/lucene2858/solr/contrib/extraction/CHANGES.txt
lucene/dev/branches/lucene2858/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
lucene/dev/branches/lucene2858/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
lucene/dev/branches/lucene2858/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
lucene/dev/branches/lucene2858/solr/contrib/uima/src/java/ (props changed)
lucene/dev/branches/lucene2858/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/branches/lucene2858/solr/core/ (props changed)
lucene/dev/branches/lucene2858/solr/core/build.xml
lucene/dev/branches/lucene2858/solr/core/src/java/ (props changed)
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/core/Config.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/CommitTracker.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/RollbackUpdateCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/UpdateLog.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java
lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java
lucene/dev/branches/lucene2858/solr/core/src/test/ (props changed)
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/schema.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/schema12.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/solrconfig-basic.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/solrconfig-nocache.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/conf/solrconfig.xml
lucene/dev/branches/lucene2858/solr/core/src/test-files/solr/solr.xml
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ZkNodePropsTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/core/TestConfig.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/core/TestLegacyMergeSchedulerPolicyConfig.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/core/TestPropInject.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
lucene/dev/branches/lucene2858/solr/dev-tools/ (props changed)
lucene/dev/branches/lucene2858/solr/example/ (props changed)
lucene/dev/branches/lucene2858/solr/example/solr/conf/schema.xml
lucene/dev/branches/lucene2858/solr/example/solr/conf/solrconfig.xml
lucene/dev/branches/lucene2858/solr/example/solr/solr.xml
lucene/dev/branches/lucene2858/solr/lib/ (props changed)
lucene/dev/branches/lucene2858/solr/lib/apache-solr-noggit-pom.xml.template
lucene/dev/branches/lucene2858/solr/scripts/ (props changed)
lucene/dev/branches/lucene2858/solr/site/ (props changed)
lucene/dev/branches/lucene2858/solr/site-src/ (props changed)
lucene/dev/branches/lucene2858/solr/solrj/ (props changed)
lucene/dev/branches/lucene2858/solr/solrj/src/java/ (props changed)
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/SolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingUpdateSolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/SolrException.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/util/FastInputStream.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/util/Hash.java
lucene/dev/branches/lucene2858/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreExampleJettyTest.java
lucene/dev/branches/lucene2858/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/branches/lucene2858/solr/test-framework/ (props changed)
lucene/dev/branches/lucene2858/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
lucene/dev/branches/lucene2858/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
lucene/dev/branches/lucene2858/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
lucene/dev/branches/lucene2858/solr/test-framework/src/java/org/apache/solr/util/AbstractSolrTestCase.java
lucene/dev/branches/lucene2858/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
lucene/dev/branches/lucene2858/solr/testlogging.properties (props changed)
lucene/dev/branches/lucene2858/solr/webapp/ (props changed)
lucene/dev/branches/lucene2858/solr/webapp/web/admin/zookeeper.jsp
lucene/dev/branches/lucene2858/solr/webapp/web/zookeeper.jsp
Modified: lucene/dev/branches/lucene2858/dev-tools/eclipse/dot.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/dev-tools/eclipse/dot.classpath?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/dev-tools/eclipse/dot.classpath (original)
+++ lucene/dev/branches/lucene2858/dev-tools/eclipse/dot.classpath Wed Jan 25 21:56:44 2012
@@ -100,7 +100,7 @@
<classpathentry kind="lib" path="modules/benchmark/lib/commons-digester-1.7.jar"/>
<classpathentry kind="lib" path="modules/benchmark/lib/commons-logging-1.0.4.jar"/>
<classpathentry kind="lib" path="modules/benchmark/lib/xercesImpl-2.9.1-patched-XERCESJ-1257.jar"/>
- <classpathentry kind="lib" path="solr/lib/apache-solr-noggit-r1209632.jar"/>
+ <classpathentry kind="lib" path="solr/lib/apache-solr-noggit-r1211150.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-fileupload-1.2.1.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-httpclient-3.1.jar"/>
@@ -115,7 +115,7 @@
<classpathentry kind="lib" path="solr/lib/slf4j-api-1.6.1.jar"/>
<classpathentry kind="lib" path="solr/lib/slf4j-jdk14-1.6.1.jar"/>
<classpathentry kind="lib" path="solr/lib/wstx-asl-3.2.7.jar"/>
- <classpathentry kind="lib" path="solr/lib/zookeeper-3.3.3.jar"/>
+ <classpathentry kind="lib" path="solr/lib/zookeeper-3.3.4.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar"/>
<classpathentry kind="lib" path="solr/example/lib/servlet-api-2.5-20081211.jar"/>
@@ -136,7 +136,7 @@
<classpathentry kind="lib" path="solr/contrib/extraction/lib/bcmail-jdk15-1.45.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/bcprov-jdk15-1.45.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/boilerpipe-1.1.0.jar"/>
- <classpathentry kind="lib" path="solr/contrib/extraction/lib/commons-compress-1.2.jar"/>
+ <classpathentry kind="lib" path="solr/contrib/extraction/lib/commons-compress-1.3.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/dom4j-1.6.1.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/fontbox-1.6.0.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/jempbox-1.6.0.jar"/>
@@ -149,8 +149,8 @@
<classpathentry kind="lib" path="solr/contrib/extraction/lib/poi-scratchpad-3.8-beta4.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/rome-0.9.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/tagsoup-1.2.1.jar"/>
- <classpathentry kind="lib" path="solr/contrib/extraction/lib/tika-core-0.10.jar"/>
- <classpathentry kind="lib" path="solr/contrib/extraction/lib/tika-parsers-0.10.jar"/>
+ <classpathentry kind="lib" path="solr/contrib/extraction/lib/tika-core-1.0.jar"/>
+ <classpathentry kind="lib" path="solr/contrib/extraction/lib/tika-parsers-1.0.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/xmlbeans-2.3.0.jar"/>
<classpathentry kind="lib" path="solr/contrib/langid/lib/langdetect-r111.jar"/>
<classpathentry kind="lib" path="solr/contrib/langid/lib/jsonic-1.2.0.jar"/>
Modified: lucene/dev/branches/lucene2858/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/dev-tools/maven/pom.xml.template?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/lucene2858/dev-tools/maven/pom.xml.template Wed Jan 25 21:56:44 2012
@@ -45,7 +45,7 @@
<jetty.version>6.1.26</jetty.version>
<patched.jetty.version>6.1.26-patched-JETTY-1340</patched.jetty.version>
<slf4j.version>1.6.1</slf4j.version>
- <tika.version>0.10</tika.version>
+ <tika.version>1.0</tika.version>
</properties>
<issueManagement>
<system>JIRA</system>
@@ -283,7 +283,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.3.3</version>
+ <version>3.3.4</version>
</dependency>
<dependency>
<groupId>org.carrot2</groupId>
@@ -670,7 +670,7 @@
<artifactId>solr-noggit</artifactId>
<version>${project.version}</version>
<packaging>jar</packaging>
- <file>solr/lib/apache-solr-noggit-r1209632.jar</file>
+ <file>solr/lib/apache-solr-noggit-r1211150.jar</file>
</configuration>
</execution>
<execution>
Modified: lucene/dev/branches/lucene2858/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/CHANGES.txt?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene2858/lucene/CHANGES.txt Wed Jan 25 21:56:44 2012
@@ -742,6 +742,9 @@ Changes in backwards compatibility polic
behavior. Added seekExact() to FSTEnum, and added FST.save/read
from a File. (Mike McCandless, Dawid Weiss, Robert Muir)
+* LUCENE-3712: Removed unused and untested ReaderUtil#subReader methods.
+ (Uwe Schindler)
+
Security fixes
* LUCENE-3588: Try harder to prevent SIGSEGV on cloned MMapIndexInputs:
@@ -814,7 +817,7 @@ Bug fixes
* LUCENE-3641: Fixed MultiReader to correctly propagate readerFinishedListeners
to clones/reopened readers. (Uwe Schindler)
-* LUCENE-3642, SOLR-2891, LUCENE-3717: Fixed bugs in CharTokenizer, n-gram filters,
+* LUCENE-3642, SOLR-2891, LUCENE-3717: Fixed bugs in CharTokenizer, n-gram tokenizers/filters,
compound token filters, thai word filter, icutokenizer, pattern analyzer,
wikipediatokenizer, and smart chinese where they would create invalid offsets in
some situations, leading to problems in highlighting.
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/analysis/Analyzer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/analysis/Analyzer.java Wed Jan 25 21:56:44 2012
@@ -52,12 +52,12 @@ public abstract class Analyzer {
* @param fieldName
* the name of the fields content passed to the
* {@link TokenStreamComponents} sink as a reader
- * @param aReader
+ * @param reader
* the reader passed to the {@link Tokenizer} constructor
* @return the {@link TokenStreamComponents} for this analyzer.
*/
protected abstract TokenStreamComponents createComponents(String fieldName,
- Reader aReader);
+ Reader reader);
/**
* Creates a TokenStream that is allowed to be re-use from the previous time
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java Wed Jan 25 21:56:44 2012
@@ -521,7 +521,7 @@ public class Lucene40PostingsReader exte
return doc = docs[i];
}
}
- return refill();
+ return doc = refill();
}
@Override
@@ -604,7 +604,7 @@ public class Lucene40PostingsReader exte
return doc = docs[i];
}
}
- return refill();
+ return doc = refill();
}
@Override
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Wed Jan 25 21:56:44 2012
@@ -548,8 +548,9 @@ class SimpleTextFieldsReader extends Fie
UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
int docID = ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length);
visitedDocs.set(docID);
- } else if (StringHelper.startsWith(scratch, POS)) {
- totalTermFreq++;
+ } else if (StringHelper.startsWith(scratch, FREQ)) {
+ UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+FREQ.length, scratch.length-FREQ.length, scratchUTF16);
+ totalTermFreq += ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length);
} else if (StringHelper.startsWith(scratch, TERM)) {
if (lastDocsStart != -1) {
b.add(Util.toIntsRef(lastTerm, scratchIntsRef), new PairOutputs.Pair<Long,PairOutputs.Pair<Long,Long>>(lastDocsStart,
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java Wed Jan 25 21:56:44 2012
@@ -32,6 +32,10 @@ public class CollectionStatistics {
private final long sumDocFreq;
public CollectionStatistics(String field, int maxDoc, int docCount, long sumTotalTermFreq, long sumDocFreq) {
+ assert maxDoc >= 0;
+ assert docCount >= -1 && docCount <= maxDoc; // #docs with field must be <= #docs
+ assert sumDocFreq >= -1;
+ assert sumTotalTermFreq == -1 || sumTotalTermFreq >= sumDocFreq; // #positions must be >= #postings
this.field = field;
this.maxDoc = maxDoc;
this.docCount = docCount;
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/TermStatistics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/TermStatistics.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/TermStatistics.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/TermStatistics.java Wed Jan 25 21:56:44 2012
@@ -29,6 +29,8 @@ public class TermStatistics {
private final long totalTermFreq;
public TermStatistics(BytesRef term, int docFreq, long totalTermFreq) {
+ assert docFreq >= 0;
+ assert totalTermFreq == -1 || totalTermFreq >= docFreq; // #positions must be >= #postings
this.term = term;
this.docFreq = docFreq;
this.totalTermFreq = totalTermFreq;
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Wed Jan 25 21:56:44 2012
@@ -87,6 +87,8 @@ public abstract class SimilarityBase ext
/** Fills all member fields defined in {@code BasicStats} in {@code stats}.
* Subclasses can override this method to fill additional stats. */
protected void fillBasicStats(BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
+ // #positions(field) must be >= #positions(term)
+ assert collectionStats.sumTotalTermFreq() == -1 || collectionStats.sumTotalTermFreq() >= termStats.totalTermFreq();
int numberOfDocuments = collectionStats.maxDoc();
int docFreq = termStats.docFreq();
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Wed Jan 25 21:56:44 2012
@@ -70,8 +70,9 @@ import org.apache.lucene.util.ToStringUt
* and 'jones' in position 1). </p>
*
* <p>Note: as {@link #getField()} returns the masked field, scoring will be
- * done using the norms of the field name supplied. This may lead to unexpected
- * scoring behaviour.</p>
+ * done using the Similarity and collection statistics of the field name supplied,
+ * but with the term statistics of the real field. This may lead to exceptions,
+ * poor performance, and unexpected scoring behaviour.</p>
*/
public class FieldMaskingSpanQuery extends SpanQuery {
private SpanQuery maskedQuery;
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Wed Jan 25 21:56:44 2012
@@ -120,7 +120,7 @@ public final class ReaderUtil {
protected abstract void add(int base, AtomicIndexReader r) throws IOException;
}
-
+
public static ReaderContext buildReaderContext(IndexReader reader) {
return new ReaderContextBuilder(reader).build();
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java Wed Jan 25 21:56:44 2012
@@ -268,8 +268,19 @@ public abstract class ShardSearchingTest
assert subStats != null;
}
- docFreq += subStats.docFreq();
- totalTermFreq += subStats.totalTermFreq();
+ int nodeDocFreq = subStats.docFreq();
+ if (docFreq >= 0 && nodeDocFreq >= 0) {
+ docFreq += nodeDocFreq;
+ } else {
+ docFreq = -1;
+ }
+
+ long nodeTotalTermFreq = subStats.totalTermFreq();
+ if (totalTermFreq >= 0 && nodeTotalTermFreq >= 0) {
+ totalTermFreq += nodeTotalTermFreq;
+ } else {
+ totalTermFreq = -1;
+ }
}
return new TermStatistics(term.bytes(), docFreq, totalTermFreq);
@@ -299,9 +310,29 @@ public abstract class ShardSearchingTest
// Collection stats are pre-shared on reopen, so,
// we better not have a cache miss:
assert nodeStats != null: "myNodeID=" + myNodeID + " nodeID=" + nodeID + " version=" + nodeVersions[nodeID] + " field=" + field;
- docCount += nodeStats.docCount();
- sumTotalTermFreq += nodeStats.sumTotalTermFreq();
- sumDocFreq += nodeStats.sumDocFreq();
+
+ int nodeDocCount = nodeStats.docCount();
+ if (docCount >= 0 && nodeDocCount >= 0) {
+ docCount += nodeDocCount;
+ } else {
+ docCount = -1;
+ }
+
+ long nodeSumTotalTermFreq = nodeStats.sumTotalTermFreq();
+ if (sumTotalTermFreq >= 0 && nodeSumTotalTermFreq >= 0) {
+ sumTotalTermFreq += nodeSumTotalTermFreq;
+ } else {
+ sumTotalTermFreq = -1;
+ }
+
+ long nodeSumDocFreq = nodeStats.sumDocFreq();
+ if (sumDocFreq >= 0 && nodeSumDocFreq >= 0) {
+ sumDocFreq += nodeSumDocFreq;
+ } else {
+ sumDocFreq = -1;
+ }
+
+ assert nodeStats.maxDoc() >= 0;
maxDoc += nodeStats.maxDoc();
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java Wed Jan 25 21:56:44 2012
@@ -31,6 +31,7 @@ import org.apache.lucene.search.CheckHit
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.AfterClass;
@@ -240,6 +241,8 @@ public class TestFieldMaskingSpanQuery e
}
public void testSimple2() throws Exception {
+ assumeTrue("Broken scoring: LUCENE-3723",
+ searcher.getSimilarityProvider().get("id") instanceof TFIDFSimilarity);
SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery q2 = new SpanTermQuery(new Term("last", "smith"));
SpanQuery q = new SpanNearQuery(new SpanQuery[]
@@ -310,6 +313,8 @@ public class TestFieldMaskingSpanQuery e
}
public void testSpans2() throws Exception {
+ assumeTrue("Broken scoring: LUCENE-3723",
+ searcher.getSimilarityProvider().get("id") instanceof TFIDFSimilarity);
SpanQuery qA1 = new SpanTermQuery(new Term("gender", "female"));
SpanQuery qA2 = new SpanTermQuery(new Term("first", "james"));
SpanQuery qA = new SpanOrQuery(qA1, new FieldMaskingSpanQuery(qA2, "gender"));
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Wed Jan 25 21:56:44 2012
@@ -74,7 +74,8 @@ public final class EdgeNGramTokenizer ex
private int gramSize;
private Side side;
private boolean started = false;
- private int inLen;
+ private int inLen; // length of the input AFTER trim()
+ private int charsRead; // length of the input
private String inStr;
@@ -183,7 +184,11 @@ public final class EdgeNGramTokenizer ex
if (!started) {
started = true;
char[] chars = new char[1024];
- int charsRead = input.read(chars);
+ charsRead = input.read(chars);
+ if (charsRead < 0) {
+ charsRead = inLen = 0;
+ return false;
+ }
inStr = new String(chars, 0, charsRead).trim(); // remove any leading or trailing spaces
inLen = inStr.length();
gramSize = minGram;
@@ -211,7 +216,7 @@ public final class EdgeNGramTokenizer ex
@Override
public final void end() {
// set final offset
- final int finalOffset = inLen;
+ final int finalOffset = correctOffset(charsRead);
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
@@ -225,5 +230,6 @@ public final class EdgeNGramTokenizer ex
public void reset() throws IOException {
super.reset();
started = false;
+ charsRead = 0;
}
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Wed Jan 25 21:56:44 2012
@@ -35,7 +35,8 @@ public final class NGramTokenizer extend
private int minGram, maxGram;
private int gramSize;
private int pos = 0;
- private int inLen;
+ private int inLen; // length of the input AFTER trim()
+ private int charsRead; // length of the input
private String inStr;
private boolean started = false;
@@ -104,7 +105,11 @@ public final class NGramTokenizer extend
started = true;
gramSize = minGram;
char[] chars = new char[1024];
- input.read(chars);
+ charsRead = input.read(chars);
+ if (charsRead < 0) {
+ charsRead = inLen = 0;
+ return false;
+ }
inStr = new String(chars).trim(); // remove any trailing empty strings
inLen = inStr.length();
}
@@ -128,7 +133,7 @@ public final class NGramTokenizer extend
@Override
public final void end() {
// set final offset
- final int finalOffset = inLen;
+ final int finalOffset = correctOffset(charsRead);
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
@@ -143,5 +148,6 @@ public final class NGramTokenizer extend
super.reset();
started = false;
pos = 0;
+ charsRead = 0;
}
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Wed Jan 25 21:56:44 2012
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.tokena
public final class PositionFilter extends TokenFilter {
/** Position increment to assign to all but the first token - default = 0 */
- private int positionIncrement = 0;
+ private final int positionIncrement;
/** The first token must have non-zero positionIncrement **/
private boolean firstTokenPositioned = false;
@@ -44,7 +44,7 @@ public final class PositionFilter extend
* @param input the input stream
*/
public PositionFilter(final TokenStream input) {
- super(input);
+ this(input, 0);
}
/**
@@ -56,7 +56,7 @@ public final class PositionFilter extend
* token from the input stream
*/
public PositionFilter(final TokenStream input, final int positionIncrement) {
- this(input);
+ super(input);
this.positionIncrement = positionIncrement;
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Wed Jan 25 21:56:44 2012
@@ -129,4 +129,27 @@ public class EdgeNGramTokenFilterTest ex
new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer,
+ new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 2, 15));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer,
+ new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 15));
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Wed Jan 25 21:56:44 2012
@@ -18,9 +18,13 @@ package org.apache.lucene.analysis.ngram
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
/**
* Tests {@link EdgeNGramTokenizer} for correctness.
@@ -95,4 +99,25 @@ public class EdgeNGramTokenizerTest exte
tokenizer.reset(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.FRONT, 2, 15);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.BACK, 2, 15);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Wed Jan 25 21:56:44 2012
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
@@ -33,89 +34,102 @@ import java.io.StringReader;
* Tests {@link NGramTokenFilter} for correctness.
*/
public class NGramTokenFilterTest extends BaseTokenStreamTestCase {
- private TokenStream input;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
- }
-
- public void testInvalidInput() throws Exception {
- boolean gotException = false;
- try {
- new NGramTokenFilter(input, 2, 1);
- } catch (IllegalArgumentException e) {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- public void testInvalidInput2() throws Exception {
- boolean gotException = false;
- try {
- new NGramTokenFilter(input, 0, 1);
- } catch (IllegalArgumentException e) {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- public void testUnigrams() throws Exception {
- NGramTokenFilter filter = new NGramTokenFilter(input, 1, 1);
- assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
- }
-
- public void testBigrams() throws Exception {
- NGramTokenFilter filter = new NGramTokenFilter(input, 2, 2);
- assertTokenStreamContents(filter, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5});
- }
-
- public void testNgrams() throws Exception {
- NGramTokenFilter filter = new NGramTokenFilter(input, 1, 3);
- assertTokenStreamContents(filter,
+ private TokenStream input;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
+ }
+
+ public void testInvalidInput() throws Exception {
+ boolean gotException = false;
+ try {
+ new NGramTokenFilter(input, 2, 1);
+ } catch (IllegalArgumentException e) {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ public void testInvalidInput2() throws Exception {
+ boolean gotException = false;
+ try {
+ new NGramTokenFilter(input, 0, 1);
+ } catch (IllegalArgumentException e) {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ public void testUnigrams() throws Exception {
+ NGramTokenFilter filter = new NGramTokenFilter(input, 1, 1);
+ assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
+ }
+
+ public void testBigrams() throws Exception {
+ NGramTokenFilter filter = new NGramTokenFilter(input, 2, 2);
+ assertTokenStreamContents(filter, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5});
+ }
+
+ public void testNgrams() throws Exception {
+ NGramTokenFilter filter = new NGramTokenFilter(input, 1, 3);
+ assertTokenStreamContents(filter,
new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"},
new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5}
- );
- }
-
- public void testOversizedNgrams() throws Exception {
- NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7);
- assertTokenStreamContents(filter, new String[0], new int[0], new int[0]);
- }
-
- public void testSmallTokenInStream() throws Exception {
- input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
- NGramTokenFilter filter = new NGramTokenFilter(input, 3, 3);
- assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
- }
-
- public void testReset() throws Exception {
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
- NGramTokenFilter filter = new NGramTokenFilter(tokenizer, 1, 1);
- assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
- tokenizer.reset(new StringReader("abcde"));
- assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
- }
-
- // LUCENE-3642
- // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
- // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
- // so in this case we behave like WDF, and preserve any modified offsets
- public void testInvalidOffsets() throws Exception {
- Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
- filters = new NGramTokenFilter(filters, 2, 2);
- return new TokenStreamComponents(tokenizer, filters);
- }
- };
- assertAnalyzesTo(analyzer, "mosfellsbær",
- new String[] { "mo", "os", "sf", "fe", "el", "ll", "ls", "sb", "ba", "ae", "er" },
- new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 });
- }
+ );
+ }
+
+ public void testOversizedNgrams() throws Exception {
+ NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7);
+ assertTokenStreamContents(filter, new String[0], new int[0], new int[0]);
+ }
+
+ public void testSmallTokenInStream() throws Exception {
+ input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
+ NGramTokenFilter filter = new NGramTokenFilter(input, 3, 3);
+ assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
+ }
+
+ public void testReset() throws Exception {
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
+ NGramTokenFilter filter = new NGramTokenFilter(tokenizer, 1, 1);
+ assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
+ tokenizer.reset(new StringReader("abcde"));
+ assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
+ }
+
+ // LUCENE-3642
+ // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
+ // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
+ // so in this case we behave like WDF, and preserve any modified offsets
+ public void testInvalidOffsets() throws Exception {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new NGramTokenFilter(filters, 2, 2);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ };
+ assertAnalyzesTo(analyzer, "mosfellsbær",
+ new String[] { "mo", "os", "sf", "fe", "el", "ll", "ls", "sb", "ba", "ae", "er" },
+ new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 });
+ }
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer,
+ new NGramTokenFilter(tokenizer, 2, 15));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Wed Jan 25 21:56:44 2012
@@ -18,71 +18,86 @@ package org.apache.lucene.analysis.ngram
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Tests {@link NGramTokenizer} for correctness.
*/
public class NGramTokenizerTest extends BaseTokenStreamTestCase {
- private StringReader input;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- input = new StringReader("abcde");
- }
-
- public void testInvalidInput() throws Exception {
- boolean gotException = false;
- try {
- new NGramTokenizer(input, 2, 1);
- } catch (IllegalArgumentException e) {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- public void testInvalidInput2() throws Exception {
- boolean gotException = false;
- try {
- new NGramTokenizer(input, 0, 1);
- } catch (IllegalArgumentException e) {
- gotException = true;
- }
- assertTrue(gotException);
- }
-
- public void testUnigrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
- assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
- }
-
- public void testBigrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(input, 2, 2);
- assertTokenStreamContents(tokenizer, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}, 5 /* abcde */);
- }
-
- public void testNgrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 3);
- assertTokenStreamContents(tokenizer,
- new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"},
- new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
- new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
- 5 /* abcde */
+ private StringReader input;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ input = new StringReader("abcde");
+ }
+
+ public void testInvalidInput() throws Exception {
+ boolean gotException = false;
+ try {
+ new NGramTokenizer(input, 2, 1);
+ } catch (IllegalArgumentException e) {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ public void testInvalidInput2() throws Exception {
+ boolean gotException = false;
+ try {
+ new NGramTokenizer(input, 0, 1);
+ } catch (IllegalArgumentException e) {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+ public void testUnigrams() throws Exception {
+ NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
+ assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
+ }
+
+ public void testBigrams() throws Exception {
+ NGramTokenizer tokenizer = new NGramTokenizer(input, 2, 2);
+ assertTokenStreamContents(tokenizer, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}, 5 /* abcde */);
+ }
+
+ public void testNgrams() throws Exception {
+ NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 3);
+ assertTokenStreamContents(tokenizer,
+ new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"},
+ new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
+ new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
+ 5 /* abcde */
);
- }
-
- public void testOversizedNgrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(input, 6, 7);
- assertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
- }
-
- public void testReset() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
- assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
- tokenizer.reset(new StringReader("abcde"));
- assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
- }
+ }
+
+ public void testOversizedNgrams() throws Exception {
+ NGramTokenizer tokenizer = new NGramTokenizer(input, 6, 7);
+ assertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
+ }
+
+ public void testReset() throws Exception {
+ NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
+ assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
+ tokenizer.reset(new StringReader("abcde"));
+ assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
+ }
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new NGramTokenizer(reader, 2, 15);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java Wed Jan 25 21:56:44 2012
@@ -17,10 +17,13 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
@@ -193,4 +196,16 @@ public class TestPathHierarchyTokenizer
new int[]{1},
path.length());
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java Wed Jan 25 21:56:44 2012
@@ -17,9 +17,13 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase {
@@ -154,4 +158,16 @@ public class TestReversePathHierarchyTok
new int[]{1, 0},
path.length());
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java Wed Jan 25 21:56:44 2012
@@ -18,14 +18,17 @@
package org.apache.lucene.analysis.pattern;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Pattern;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Tests {@link PatternReplaceCharFilter}
@@ -172,4 +175,21 @@ public class TestPatternReplaceCharFilte
private Pattern pattern( String p ){
return Pattern.compile( p );
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(Reader reader) {
+ return new PatternReplaceCharFilter(Pattern.compile("a"), "b", CharReader.get(reader));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java Wed Jan 25 21:56:44 2012
@@ -17,10 +17,13 @@
package org.apache.lucene.analysis.pattern;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Pattern;
@@ -77,5 +80,28 @@ public class TestPatternReplaceFilter ex
assertTokenStreamContents(ts,
new String[] { "aa$fooaa$fooa$foo$", "a$", "caaaaaaaaa$" });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
+ return new TokenStreamComponents(tokenizer, filter);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
+ return new TokenStreamComponents(tokenizer, filter);
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Wed Jan 25 21:56:44 2012
@@ -18,17 +18,22 @@
package org.apache.lucene.analysis.pattern;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TestPatternTokenizer extends BaseTokenStreamTestCase
@@ -117,4 +122,35 @@ public class TestPatternTokenizer extend
in.close();
return out.toString();
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = null;
+ try {
+ tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), -1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = null;
+ try {
+ tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), 0);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java Wed Jan 25 21:56:44 2012
@@ -17,11 +17,14 @@
package org.apache.lucene.analysis.reverse;
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util.Version;
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
@@ -96,4 +99,16 @@ public class TestReverseStringFilter ext
ReverseStringFilter.reverse(TEST_VERSION_CURRENT, buffer, 3, 7);
assertEquals("abcfedð©¬
æ¯ç", new String(buffer));
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=1235969&r1=1235968&r2=1235969&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/dev/branches/lucene2858/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Wed Jan 25 21:56:44 2012
@@ -18,9 +18,12 @@ package org.apache.lucene.analysis.shing
*/
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -1129,4 +1132,16 @@ public class ShingleFilterTest extends B
token.setPositionIncrement(positionIncrement);
return token;
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}