You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/12/07 12:37:35 UTC
svn commit: r1643662 [1/6] - in /lucene/dev/branches/lucene6005: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/co...
Author: mikemccand
Date: Sun Dec 7 11:37:32 2014
New Revision: 1643662
URL: http://svn.apache.org/r1643662
Log:
LUCENE-6005: merge trunk
Added:
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/AbortingException.java
- copied unchanged from r1643659, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AbortingException.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
- copied, changed from r1643659, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java
- copied, changed from r1643659, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
- copied, changed from r1643659, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
lucene/dev/branches/lucene6005/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
- copied unchanged from r1643659, lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java
- copied unchanged from r1643659, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/SuppressingConcurrentMergeScheduler.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
- copied unchanged from r1643659, lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
- copied unchanged from r1643659, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
- copied unchanged from r1643659, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
Removed:
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterAbort.java
lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/SolrConfigRestApi.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/DefaultSearchFieldResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SchemaNameResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SchemaSimilarityResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SchemaVersionResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SchemaZkVersionResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SolrQueryParserDefaultOperatorResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/SolrQueryParserResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/schema/UniqueKeyFieldResource.java
Modified:
lucene/dev/branches/lucene6005/ (props changed)
lucene/dev/branches/lucene6005/lucene/ (props changed)
lucene/dev/branches/lucene6005/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/lucene/codecs/ (props changed)
lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/branches/lucene6005/lucene/core/ (props changed)
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfoFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
lucene/dev/branches/lucene6005/lucene/facet/ (props changed)
lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
lucene/dev/branches/lucene6005/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene6005/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
lucene/dev/branches/lucene6005/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
lucene/dev/branches/lucene6005/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/branches/lucene6005/lucene/memory/ (props changed)
lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/lucene6005/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
lucene/dev/branches/lucene6005/lucene/misc/ (props changed)
lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
lucene/dev/branches/lucene6005/lucene/queries/ (props changed)
lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java
lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
lucene/dev/branches/lucene6005/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
lucene/dev/branches/lucene6005/lucene/spatial/ (props changed)
lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java
lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java
lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java
lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/branches/lucene6005/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyStoredFieldsFormat.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyTermVectorsFormat.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/mockfile/VerboseFS.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
lucene/dev/branches/lucene6005/solr/ (props changed)
lucene/dev/branches/lucene6005/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/solr/bin/ (props changed)
lucene/dev/branches/lucene6005/solr/bin/solr
lucene/dev/branches/lucene6005/solr/bin/solr.cmd
lucene/dev/branches/lucene6005/solr/bin/solr.in.cmd
lucene/dev/branches/lucene6005/solr/bin/solr.in.sh
lucene/dev/branches/lucene6005/solr/build.xml (contents, props changed)
lucene/dev/branches/lucene6005/solr/cloud-dev/ (props changed)
lucene/dev/branches/lucene6005/solr/cloud-dev/clean.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/cli-test-solrcloud-start.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/functions.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/solrcloud-extzk-start.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/solrcloud-multi-start.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/solrcloud-start-existing.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/solrcloud-start.sh
lucene/dev/branches/lucene6005/solr/cloud-dev/stop.sh
lucene/dev/branches/lucene6005/solr/contrib/ (props changed)
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java
lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
lucene/dev/branches/lucene6005/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java
lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/core/ (props changed)
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/RestManager.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java
lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/schema11.xml
lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/SolrRestletTestBase.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/TestRestManager.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java
lucene/dev/branches/lucene6005/solr/solrj/ (props changed)
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/params/DefaultSolrParams.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/util/IteratorChain.java
lucene/dev/branches/lucene6005/solr/solrj/src/test/org/apache/solr/common/params/SolrParamTest.java
lucene/dev/branches/lucene6005/solr/webapp/ (props changed)
lucene/dev/branches/lucene6005/solr/webapp/build.xml
lucene/dev/branches/lucene6005/solr/webapp/web/WEB-INF/web.xml
Modified: lucene/dev/branches/lucene6005/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/CHANGES.txt?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6005/lucene/CHANGES.txt Sun Dec 7 11:37:32 2014
@@ -117,6 +117,61 @@ New Features
* LUCENE-6077: Added a filter cache. (Adrien Grand, Robert Muir)
+* LUCENE-6088: TermsFilter implements Accountable. (Adrien Grand)
+
+* LUCENE-6034: The default highlighter when used with QueryScorer will highlight payload-sensitive
+ queries provided that term vectors with positions, offsets, and payloads are present. This is the
+ only highlighter that can highlight such queries accurately. (David Smiley)
+
+* LUCENE-5914: Add an option to Lucene50Codec to support either BEST_SPEED
+ or BEST_COMPRESSION for stored fields. (Adrien Grand, Robert Muir)
+
+Optimizations
+
+* LUCENE-5960: Use a more efficient bitset, not a Set<Integer>, to
+ track visited states. (Markus Heiden via Mike McCandless)
+
+* LUCENE-5959: Don't allocate excess memory when building automaton in
+ finish. (Markus Heiden via Mike McCandless)
+
+* LUCENE-5963: Reduce memory allocations in
+ AnalyzingSuggester. (Markus Heiden via Mike McCandless)
+
+* LUCENE-5938: MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is now faster on
+ queries that match few documents by using a sparse bit set implementation.
+ (Adrien Grand)
+
+* LUCENE-5969: Refactor merging to be more efficient, checksum calculation is
+ per-segment/per-producer, and norms and doc values merging no longer cause
+ RAM spikes for latent fields. (Mike McCandless, Robert Muir)
+
+* LUCENE-5983: CachingWrapperFilter now uses a new DocIdSet implementation
+ called RoaringDocIdSet instead of WAH8DocIdSet. (Adrien Grand)
+
+* LUCENE-6022: DocValuesDocIdSet checks live docs before doc values.
+ (Adrien Grand)
+
+* LUCENE-6030: Add norms patched compression for a small number of common values
+ (Ryan Ernst)
+
+* LUCENE-6040: Speed up EliasFanoDocIdSet through broadword bit selection.
+ (Paul Elschot)
+
+* LUCENE-6033: CachingTokenFilter now uses ArrayList not LinkedList, and has new
+ isCached() method. (David Smiley)
+
+* LUCENE-6031: TokenSources (in the default highlighter) converts term vectors into a
+ TokenStream much faster in linear time (not N*log(N) using less memory, and with reset()
+ implemented. Only one of offsets or positions are required of the term vector.
+ (David Smiley)
+
+* LUCENE-6089, LUCENE-6090: Tune CompressionMode.HIGH_COMPRESSION for
+ better compression and less cpu usage. (Adrien Grand, Robert Muir)
+
+* LUCENE-6034: QueryScorer, used by the default highlighter, needn't re-index the provided
+ TokenStream with MemoryIndex when it comes from TokenSources (term vectors) with offsets and
+ positions. (David Smiley)
+
API Changes
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
@@ -227,6 +282,19 @@ API Changes
* LUCENE-6068: LeafReader.fields() never returns null. (Robert Muir)
+* LUCENE-6082: Remove abort() from codec apis. (Robert Muir)
+
+* LUCENE-6084: IndexOutput's constructor now requires a String
+ resourceDescription so its toString is sane (Robert Muir, Mike
+ McCandless)
+
+* LUCENE-6087: Allow passing custom DirectoryReader to SearcherManager
+ (Mike McCandless)
+
+* LUCENE-6085: Undeprecate SegmentInfo attributes, but add safety so they
+ won't be trappy if codec tries to use them during docvalues updates.
+ (Robert Muir)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
@@ -286,6 +354,13 @@ Bug Fixes
* LUCENE-6075: Don't overflow int in SimpleRateLimiter (Boaz Leskes
via Mike McCandless)
+
+* LUCENE-5987: IndexWriter will now forcefully close itself on
+ aborting exception (an exception that would otherwise cause silent
+ data loss). (Robert Muir, Mike McCandless)
+
+* LUCENE-6094: Allow IW.rollback to stop ConcurrentMergeScheduler even
+ when it's stalling because there are too many merges. (Mike McCandless)
Documentation
@@ -316,45 +391,6 @@ Tests
* LUCENE-5968: Improve error message when 'ant beast' is run on top-level
modules. (Ramkumar Aiyengar, Uwe Schindler)
-Optimizations
-
-* LUCENE-5960: Use a more efficient bitset, not a Set<Integer>, to
- track visited states. (Markus Heiden via Mike McCandless)
-
-* LUCENE-5959: Don't allocate excess memory when building automaton in
- finish. (Markus Heiden via Mike McCandless)
-
-* LUCENE-5963: Reduce memory allocations in
- AnalyzingSuggester. (Markus Heiden via Mike McCandless)
-
-* LUCENE-5938: MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is now faster on
- queries that match few documents by using a sparse bit set implementation.
- (Adrien Grand)
-
-* LUCENE-5969: Refactor merging to be more efficient, checksum calculation is
- per-segment/per-producer, and norms and doc values merging no longer cause
- RAM spikes for latent fields. (Mike McCandless, Robert Muir)
-
-* LUCENE-5983: CachingWrapperFilter now uses a new DocIdSet implementation
- called RoaringDocIdSet instead of WAH8DocIdSet. (Adrien Grand)
-
-* LUCENE-6022: DocValuesDocIdSet checks live docs before doc values.
- (Adrien Grand)
-
-* LUCENE-6030: Add norms patched compression for a small number of common values
- (Ryan Ernst)
-
-* LUCENE-6040: Speed up EliasFanoDocIdSet through broadword bit selection.
- (Paul Elschot)
-
-* LUCENE-6033: CachingTokenFilter now uses ArrayList not LinkedList, and has new
- isCached() method. (David Smiley)
-
-* LUCENE-6031: TokenSources (in the default highlighter) converts term vectors into a
- TokenStream much faster in linear time (not N*log(N) using less memory, and with reset()
- implemented. Only one of offsets or positions are required of the term vector.
- (David Smiley)
-
Build
* LUCENE-5909: Smoke tester now has better command line parsing and
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java Sun Dec 7 11:37:32 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -36,7 +37,6 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
@@ -53,6 +53,9 @@ public class SimpleTextSegmentInfoFormat
final static BytesRef SI_NUM_DIAG = new BytesRef(" diagnostics ");
final static BytesRef SI_DIAG_KEY = new BytesRef(" key ");
final static BytesRef SI_DIAG_VALUE = new BytesRef(" value ");
+ final static BytesRef SI_NUM_ATT = new BytesRef(" attributes ");
+ final static BytesRef SI_ATT_KEY = new BytesRef(" key ");
+ final static BytesRef SI_ATT_VALUE = new BytesRef(" value ");
final static BytesRef SI_NUM_FILES = new BytesRef(" files ");
final static BytesRef SI_FILE = new BytesRef(" file ");
final static BytesRef SI_ID = new BytesRef(" id ");
@@ -63,9 +66,7 @@ public class SimpleTextSegmentInfoFormat
public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException {
BytesRefBuilder scratch = new BytesRefBuilder();
String segFileName = IndexFileNames.segmentFileName(segmentName, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
- ChecksumIndexInput input = directory.openChecksumInput(segFileName, context);
- boolean success = false;
- try {
+ try (ChecksumIndexInput input = directory.openChecksumInput(segFileName, context)) {
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SI_VERSION);
final Version version;
@@ -100,6 +101,22 @@ public class SimpleTextSegmentInfoFormat
}
SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_NUM_ATT);
+ int numAtt = Integer.parseInt(readString(SI_NUM_ATT.length, scratch));
+ Map<String,String> attributes = new HashMap<>(numAtt);
+
+ for (int i = 0; i < numAtt; i++) {
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_ATT_KEY);
+ String key = readString(SI_ATT_KEY.length, scratch);
+
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_ATT_VALUE);
+ String value = readString(SI_ATT_VALUE.length, scratch);
+ attributes.put(key, value);
+ }
+
+ SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SI_NUM_FILES);
int numFiles = Integer.parseInt(readString(SI_NUM_FILES.length, scratch));
Set<String> files = new HashSet<>();
@@ -123,16 +140,9 @@ public class SimpleTextSegmentInfoFormat
SimpleTextUtil.checkFooter(input);
SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount,
- isCompoundFile, null, diagnostics, id);
+ isCompoundFile, null, diagnostics, id, Collections.unmodifiableMap(attributes));
info.setFiles(files);
- success = true;
return info;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input);
- } else {
- input.close();
- }
}
}
@@ -146,10 +156,7 @@ public class SimpleTextSegmentInfoFormat
String segFileName = IndexFileNames.segmentFileName(si.name, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
si.addFile(segFileName);
- boolean success = false;
- IndexOutput output = dir.createOutput(segFileName, ioContext);
-
- try {
+ try (IndexOutput output = dir.createOutput(segFileName, ioContext)) {
BytesRefBuilder scratch = new BytesRefBuilder();
SimpleTextUtil.write(output, SI_VERSION);
@@ -182,6 +189,21 @@ public class SimpleTextSegmentInfoFormat
}
}
+ Map<String,String> attributes = si.getAttributes();
+ SimpleTextUtil.write(output, SI_NUM_ATT);
+ SimpleTextUtil.write(output, Integer.toString(attributes.size()), scratch);
+ SimpleTextUtil.writeNewline(output);
+
+ for (Map.Entry<String,String> attEntry : attributes.entrySet()) {
+ SimpleTextUtil.write(output, SI_ATT_KEY);
+ SimpleTextUtil.write(output, attEntry.getKey(), scratch);
+ SimpleTextUtil.writeNewline(output);
+
+ SimpleTextUtil.write(output, SI_ATT_VALUE);
+ SimpleTextUtil.write(output, attEntry.getValue(), scratch);
+ SimpleTextUtil.writeNewline(output);
+ }
+
Set<String> files = si.files();
int numFiles = files == null ? 0 : files.size();
SimpleTextUtil.write(output, SI_NUM_FILES);
@@ -201,14 +223,6 @@ public class SimpleTextSegmentInfoFormat
SimpleTextUtil.writeNewline(output);
SimpleTextUtil.writeChecksum(output, scratch);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(output);
- IOUtils.deleteFilesIgnoringExceptions(dir, segFileName);
- } else {
- output.close();
- }
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java Sun Dec 7 11:37:32 2014
@@ -70,7 +70,7 @@ public class SimpleTextStoredFieldsWrite
success = true;
} finally {
if (!success) {
- abort();
+ IOUtils.closeWhileHandlingException(this);
}
}
}
@@ -152,14 +152,6 @@ public class SimpleTextStoredFieldsWrite
}
@Override
- public void abort() {
- try {
- close();
- } catch (Throwable ignored) {}
- IOUtils.deleteFilesIgnoringExceptions(directory, IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION));
- }
-
- @Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numDocsWritten != numDocs) {
throw new RuntimeException("mergeFields produced an invalid result: docCount is " + numDocs
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java Sun Dec 7 11:37:32 2014
@@ -74,7 +74,7 @@ public class SimpleTextTermVectorsWriter
success = true;
} finally {
if (!success) {
- abort();
+ IOUtils.closeWhileHandlingException(this);
}
}
}
@@ -164,14 +164,6 @@ public class SimpleTextTermVectorsWriter
}
@Override
- public void abort() {
- try {
- close();
- } catch (Throwable ignored) {}
- IOUtils.deleteFilesIgnoringExceptions(directory, IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION));
- }
-
- @Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numDocsWritten != numDocs) {
throw new RuntimeException("mergeVectors produced an invalid result: mergedDocs is " + numDocs + " but vec numDocs is " + numDocsWritten + " file=" + out.toString() + "; now aborting this merge to prevent index corruption");
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfoFormat.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfoFormat.java Sun Dec 7 11:37:32 2014
@@ -48,7 +48,8 @@ public abstract class SegmentInfoFormat
public abstract SegmentInfo read(Directory directory, String segmentName, byte segmentID[], IOContext context) throws IOException;
/**
- * Write {@link SegmentInfo} data.
+ * Write {@link SegmentInfo} data.
+ * The codec must add its SegmentInfo filename(s) to {@code info} before doing i/o.
* @throws IOException If an I/O error occurs
*/
public abstract void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Sun Dec 7 11:37:32 2014
@@ -60,10 +60,6 @@ public abstract class StoredFieldsWriter
/** Writes a single stored field. */
public abstract void writeField(FieldInfo info, IndexableField field) throws IOException;
-
- /** Aborts writing entirely, implementation should remove
- * any partially-written files, etc. */
- public abstract void abort();
/** Called before {@link #close()}, passing in the number
* of documents that were written. Note that this is
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Sun Dec 7 11:37:32 2014
@@ -93,10 +93,6 @@ public abstract class TermVectorsWriter
/** Adds a term position and offsets */
public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
-
- /** Aborts writing entirely, implementation should remove
- * any partially-written files, etc. */
- public abstract void abort();
/** Called before {@link #close()}, passing in the number
* of documents that were written. Note that this is
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Sun Dec 7 11:37:32 2014
@@ -48,15 +48,16 @@ public class CompressingStoredFieldsForm
private final String segmentSuffix;
private final CompressionMode compressionMode;
private final int chunkSize;
+ private final int maxDocsPerChunk;
/**
* Create a new {@link CompressingStoredFieldsFormat} with an empty segment
* suffix.
*
- * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int)
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int, int)
*/
- public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
- this(formatName, "", compressionMode, chunkSize);
+ public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
+ this(formatName, "", compressionMode, chunkSize, maxDocsPerChunk);
}
/**
@@ -79,6 +80,8 @@ public class CompressingStoredFieldsForm
* <code>chunkSize</code> is the minimum byte size of a chunk of documents.
* A value of <code>1</code> can make sense if there is redundancy across
* fields.
+ * <code>maxDocsPerChunk</code> is an upperbound on how many docs may be stored
+ * in a single chunk. This is to bound the cpu costs for highly compressible data.
* <p>
* Higher values of <code>chunkSize</code> should improve the compression
* ratio but will require more memory at indexing time and might make document
@@ -88,10 +91,11 @@ public class CompressingStoredFieldsForm
* @param formatName the name of the {@link StoredFieldsFormat}
* @param compressionMode the {@link CompressionMode} to use
* @param chunkSize the minimum number of bytes of a single chunk of stored documents
+ * @param maxDocsPerChunk the maximum number of documents in a single chunk
* @see CompressionMode
*/
public CompressingStoredFieldsFormat(String formatName, String segmentSuffix,
- CompressionMode compressionMode, int chunkSize) {
+ CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
this.formatName = formatName;
this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
@@ -99,7 +103,10 @@ public class CompressingStoredFieldsForm
throw new IllegalArgumentException("chunkSize must be >= 1");
}
this.chunkSize = chunkSize;
-
+ if (maxDocsPerChunk < 1) {
+ throw new IllegalArgumentException("maxDocsPerChunk must be >= 1");
+ }
+ this.maxDocsPerChunk = maxDocsPerChunk;
}
@Override
@@ -113,13 +120,13 @@ public class CompressingStoredFieldsForm
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
IOContext context) throws IOException {
return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
- formatName, compressionMode, chunkSize);
+ formatName, compressionMode, chunkSize, maxDocsPerChunk);
}
@Override
public String toString() {
return getClass().getSimpleName() + "(compressionMode=" + compressionMode
- + ", chunkSize=" + chunkSize + ")";
+ + ", chunkSize=" + chunkSize + ", maxDocsPerChunk=" + maxDocsPerChunk + ")";
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Sun Dec 7 11:37:32 2014
@@ -54,9 +54,6 @@ public final class CompressingStoredFiel
/** Extension of stored fields index file */
public static final String FIELDS_INDEX_EXTENSION = "fdx";
-
- // hard limit on the maximum number of documents per chunk
- static final int MAX_DOCUMENTS_PER_CHUNK = 128;
static final int STRING = 0x00;
static final int BYTE_ARR = 0x01;
@@ -82,6 +79,7 @@ public final class CompressingStoredFiel
private final CompressionMode compressionMode;
private final Compressor compressor;
private final int chunkSize;
+ private final int maxDocsPerChunk;
private final GrowableByteArrayDataOutput bufferedDocs;
private int[] numStoredFields; // number of stored fields
@@ -91,7 +89,7 @@ public final class CompressingStoredFiel
/** Sole constructor. */
public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context,
- String formatName, CompressionMode compressionMode, int chunkSize) throws IOException {
+ String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) throws IOException {
assert directory != null;
this.directory = directory;
this.segment = si.name;
@@ -99,6 +97,7 @@ public final class CompressingStoredFiel
this.compressionMode = compressionMode;
this.compressor = compressionMode.newCompressor();
this.chunkSize = chunkSize;
+ this.maxDocsPerChunk = maxDocsPerChunk;
this.docBase = 0;
this.bufferedDocs = new GrowableByteArrayDataOutput(chunkSize);
this.numStoredFields = new int[16];
@@ -128,8 +127,7 @@ public final class CompressingStoredFiel
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(indexStream);
- abort();
+ IOUtils.closeWhileHandlingException(fieldsStream, indexStream, indexWriter);
}
}
}
@@ -211,7 +209,7 @@ public final class CompressingStoredFiel
private boolean triggerFlush() {
return bufferedDocs.length >= chunkSize || // chunks of at least chunkSize bytes
- numBufferedDocs >= MAX_DOCUMENTS_PER_CHUNK;
+ numBufferedDocs >= maxDocsPerChunk;
}
private void flush() throws IOException {
@@ -308,14 +306,6 @@ public final class CompressingStoredFiel
}
@Override
- public void abort() {
- IOUtils.closeWhileHandlingException(this);
- IOUtils.deleteFilesIgnoringExceptions(directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION));
- }
-
- @Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numBufferedDocs > 0) {
flush();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java Sun Dec 7 11:37:32 2014
@@ -248,8 +248,7 @@ public final class CompressingTermVector
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(indexStream);
- abort();
+ IOUtils.closeWhileHandlingException(vectorsStream, indexStream, indexWriter);
}
}
}
@@ -265,14 +264,6 @@ public final class CompressingTermVector
}
@Override
- public void abort() {
- IOUtils.closeWhileHandlingException(this);
- IOUtils.deleteFilesIgnoringExceptions(directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION));
- }
-
- @Override
public void startDocument(int numVectorFields) throws IOException {
curDoc = addDocData(numVectorFields);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java Sun Dec 7 11:37:32 2014
@@ -70,7 +70,8 @@ public abstract class CompressionMode {
@Override
public Compressor newCompressor() {
- return new DeflateCompressor(Deflater.BEST_COMPRESSION);
+ // 3 is the highest level that doesn't have lazy match evaluation
+ return new DeflateCompressor(3);
}
@Override
@@ -185,7 +186,7 @@ public abstract class CompressionMode {
byte[] compressed;
DeflateDecompressor() {
- decompressor = new Inflater();
+ decompressor = new Inflater(true);
compressed = new byte[0];
}
@@ -197,13 +198,18 @@ public abstract class CompressionMode {
return;
}
final int compressedLength = in.readVInt();
- if (compressedLength > compressed.length) {
- compressed = new byte[ArrayUtil.oversize(compressedLength, 1)];
+ // pad with extra "dummy byte": see javadocs for using Inflater(true)
+ // we do it for compliance, but its unnecessary for years in zlib.
+ final int paddedLength = compressedLength + 1;
+ if (paddedLength > compressed.length) {
+ compressed = new byte[ArrayUtil.oversize(paddedLength, 1)];
}
in.readBytes(compressed, 0, compressedLength);
+ compressed[compressedLength] = 0; // explicitly set dummy byte to 0
decompressor.reset();
- decompressor.setInput(compressed, 0, compressedLength);
+ // extra "dummy byte"
+ decompressor.setInput(compressed, 0, paddedLength);
bytes.offset = bytes.length = 0;
while (true) {
@@ -241,7 +247,7 @@ public abstract class CompressionMode {
byte[] compressed;
DeflateCompressor(int level) {
- compressor = new Deflater(level);
+ compressor = new Deflater(level, true);
compressed = new byte[64];
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java Sun Dec 7 11:37:32 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.codecs.lucene5
* limitations under the License.
*/
+import java.util.Objects;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
@@ -28,6 +30,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -42,7 +45,6 @@ import org.apache.lucene.codecs.perfield
* @lucene.experimental
*/
public class Lucene50Codec extends Codec {
- private final StoredFieldsFormat fieldsFormat = new Lucene50StoredFieldsFormat();
private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene50FieldInfosFormat();
private final SegmentInfoFormat segmentInfosFormat = new Lucene50SegmentInfoFormat();
@@ -62,15 +64,30 @@ public class Lucene50Codec extends Codec
return Lucene50Codec.this.getDocValuesFormatForField(field);
}
};
+
+ private final StoredFieldsFormat storedFieldsFormat;
- /** Sole constructor. */
+ /**
+ * Instantiates a new codec.
+ */
public Lucene50Codec() {
+ this(Mode.BEST_SPEED);
+ }
+
+ /**
+ * Instantiates a new codec, specifying the stored fields compression
+ * mode to use.
+ * @param mode stored fields compression mode to use for newly
+ * flushed/merged segments.
+ */
+ public Lucene50Codec(Mode mode) {
super("Lucene50");
+ this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));
}
@Override
public final StoredFieldsFormat storedFieldsFormat() {
- return fieldsFormat;
+ return storedFieldsFormat;
}
@Override
@@ -106,7 +123,11 @@ public class Lucene50Codec extends Codec
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
*
- * The default implementation always returns "Lucene50"
+ * The default implementation always returns "Lucene50".
+ * <p>
+ * <b>WARNING:</b> if you subclass, you are responsible for index
+ * backwards compatibility: future version of Lucene are only
+ * guaranteed to be able to read the default implementation.
*/
public PostingsFormat getPostingsFormatForField(String field) {
return defaultFormat;
@@ -115,7 +136,11 @@ public class Lucene50Codec extends Codec
/** Returns the docvalues format that should be used for writing
* new segments of <code>field</code>.
*
- * The default implementation always returns "Lucene50"
+ * The default implementation always returns "Lucene50".
+ * <p>
+ * <b>WARNING:</b> if you subclass, you are responsible for index
+ * backwards compatibility: future version of Lucene are only
+ * guaranteed to be able to read the default implementation.
*/
public DocValuesFormat getDocValuesFormatForField(String field) {
return defaultDVFormat;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java Sun Dec 7 11:37:32 2014
@@ -221,7 +221,7 @@ public final class Lucene50PostingsWrite
final int docDelta = docID - lastDocID;
if (docID < 0 || (docCount > 0 && docDelta <= 0)) {
- throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )", docOut.toString());
+ throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
}
docDeltaBuffer[docBufferUpto] = docDelta;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java Sun Dec 7 11:37:32 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene5
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.Map;
import java.util.Set;
@@ -33,7 +34,6 @@ import org.apache.lucene.store.DataOutpu
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
@@ -41,7 +41,7 @@ import org.apache.lucene.util.Version;
* <p>
* Files:
* <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Attributes, Footer
* </ul>
* </p>
* Data types:
@@ -51,7 +51,7 @@ import org.apache.lucene.util.Version;
* <li>SegSize --> {@link DataOutput#writeInt Int32}</li>
* <li>SegVersion --> {@link DataOutput#writeString String}</li>
* <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
- * <li>Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
+ * <li>Diagnostics,Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
@@ -101,8 +101,9 @@ public class Lucene50SegmentInfoFormat e
final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
+ final Map<String,String> attributes = input.readStringStringMap();
- si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID);
+ si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, Collections.unmodifiableMap(attributes));
si.setFiles(files);
} catch (Throwable exception) {
priorE = exception;
@@ -118,7 +119,6 @@ public class Lucene50SegmentInfoFormat e
final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene50SegmentInfoFormat.SI_EXTENSION);
si.addFile(fileName);
- boolean success = false;
try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
CodecUtil.writeIndexHeader(output,
Lucene50SegmentInfoFormat.CODEC_NAME,
@@ -145,13 +145,8 @@ public class Lucene50SegmentInfoFormat e
}
}
output.writeStringSet(files);
+ output.writeStringStringMap(si.getAttributes());
CodecUtil.writeFooter(output);
- success = true;
- } finally {
- if (!success) {
- // TODO: are we doing this outside of the tracking wrapper? why must SIWriter cleanup like this?
- IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
- }
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java Sun Dec 7 11:37:32 2014
@@ -17,26 +17,46 @@ package org.apache.lucene.codecs.lucene5
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Objects;
+
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexWriter;
import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.packed.PackedInts;
/**
* Lucene 5.0 stored fields format.
*
* <p><b>Principle</b></p>
- * <p>This {@link StoredFieldsFormat} compresses blocks of 16KB of documents in
+ * <p>This {@link StoredFieldsFormat} compresses blocks of documents in
* order to improve the compression ratio compared to document-level
* compression. It uses the <a href="http://code.google.com/p/lz4/">LZ4</a>
- * compression algorithm, which is fast to compress and very fast to decompress
- * data. Although the compression method that is used focuses more on speed
- * than on compression ratio, it should provide interesting compression ratios
- * for redundant inputs (such as log files, HTML or plain text).</p>
+ * compression algorithm by default in 16KB blocks, which is fast to compress
+ * and very fast to decompress data. Although the default compression method
+ * that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on
+ * compression ratio, it should provide interesting compression ratios
+ * for redundant inputs (such as log files, HTML or plain text). For higher
+ * compression, you can choose ({@link Mode#BEST_COMPRESSION BEST_COMPRESSION}), which uses
+ * the <a href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a> algorithm with 24KB blocks
+ * for a better ratio at the expense of slower performance.
+ * These two options can be configured like this: </p>
+ * <pre class="prettyprint">
+ * // the default: for high performance
+ * indexWriterConfig.setCodec(new Lucene50Codec(Mode.BEST_SPEED));
+ * // instead for higher performance (but slower):
+ * // indexWriterConfig.setCodec(new Lucene50Codec(Mode.BEST_COMPRESSION));
+ * </pre>
* <p><b>File formats</b></p>
* <p>Stored fields are represented by two files:</p>
* <ol>
@@ -114,11 +134,58 @@ import org.apache.lucene.util.packed.Pac
* larger than (<tt>2<sup>31</sup> - 2<sup>14</sup></tt>) bytes.</p>
* @lucene.experimental
*/
-public final class Lucene50StoredFieldsFormat extends CompressingStoredFieldsFormat {
-
- /** Sole constructor. */
+public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
+
+ /** Configuration option for stored fields. */
+ public static enum Mode {
+ /** Trade compression ratio for retrieval speed. */
+ BEST_SPEED,
+ /** Trade retrieval speed for compression ratio. */
+ BEST_COMPRESSION
+ }
+
+ /** Attribute key for compression mode. */
+ public static final String MODE_KEY = Lucene50StoredFieldsFormat.class.getSimpleName() + ".mode";
+
+ final Mode mode;
+
+ /** Stored fields format with default options */
public Lucene50StoredFieldsFormat() {
- super("Lucene50StoredFields", CompressionMode.FAST, 1 << 14);
+ this(Mode.BEST_SPEED);
+ }
+
+ /** Stored fields format with specified mode */
+ public Lucene50StoredFieldsFormat(Mode mode) {
+ this.mode = Objects.requireNonNull(mode);
}
+ @Override
+ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
+ String value = si.getAttribute(MODE_KEY);
+ if (value == null) {
+ throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
+ }
+ Mode mode = Mode.valueOf(value);
+ return impl(mode).fieldsReader(directory, si, fn, context);
+ }
+
+ @Override
+ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
+ String previous = si.putAttribute(MODE_KEY, mode.name());
+ if (previous != null) {
+ throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
+ "old=" + previous + ", new=" + mode.name());
+ }
+ return impl(mode).fieldsWriter(directory, si, context);
+ }
+
+ StoredFieldsFormat impl(Mode mode) {
+ switch (mode) {
+ case BEST_SPEED:
+ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsFast", CompressionMode.FAST, 1 << 14, 128);
+ case BEST_COMPRESSION:
+ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsHigh", CompressionMode.HIGH_COMPRESSION, 24576, 512);
+ default: throw new AssertionError();
+ }
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Sun Dec 7 11:37:32 2014
@@ -606,6 +606,10 @@ public class FieldTypes {
}
// nocommit must check that if fastRanges is on, you have a PF that supports it
+
+ if (fastRanges == Boolean.TRUE && indexOptions != IndexOptions.DOCS) {
+ illegalState(name, "fastRanges is only possible when indexOptions=DOCS; got: " + indexOptions);
+ }
if (analyzerPositionGap != null) {
if (indexOptions == IndexOptions.NONE) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sun Dec 7 11:37:32 2014
@@ -300,8 +300,11 @@ public class ConcurrentMergeScheduler ex
protected synchronized int mergeThreadCount() {
int count = 0;
for (MergeThread mt : mergeThreads) {
- if (mt.isAlive() && mt.getCurrentMerge() != null) {
- count++;
+ if (mt.isAlive()) {
+ MergePolicy.OneMerge merge = mt.getCurrentMerge();
+ if (merge != null && merge.isAborted() == false) {
+ count++;
+ }
}
}
return count;
@@ -399,7 +402,8 @@ public class ConcurrentMergeScheduler ex
message(" too many merges; stalling...");
}
try {
- wait();
+ // Only wait 0.25 seconds, so if all merges are aborted (by IW.rollback) we notice:
+ wait(250);
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java Sun Dec 7 11:37:32 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
/**
* This exception is thrown when Lucene detects
@@ -31,11 +32,21 @@ public class CorruptIndexException exten
public CorruptIndexException(String message, DataInput input) {
this(message, input, null);
}
+
+ /** Create exception with a message only */
+ public CorruptIndexException(String message, DataOutput output) {
+ this(message, output, null);
+ }
/** Create exception with message and root cause. */
public CorruptIndexException(String message, DataInput input, Throwable cause) {
this(message, Objects.toString(input), cause);
}
+
+ /** Create exception with message and root cause. */
+ public CorruptIndexException(String message, DataOutput output, Throwable cause) {
+ this(message, Objects.toString(output), cause);
+ }
/** Create exception with a message only */
public CorruptIndexException(String message, String resourceDescription) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sun Dec 7 11:37:32 2014
@@ -82,7 +82,7 @@ final class DefaultIndexingChain extends
}
@Override
- public void flush(SegmentWriteState state) throws IOException {
+ public void flush(SegmentWriteState state) throws IOException, AbortingException {
// NOTE: caller (DocumentsWriterPerThread) handles
// aborting on any exception from this method
@@ -176,7 +176,7 @@ final class DefaultIndexingChain extends
/** Catch up for all docs before us that had no stored
* fields, or hit non-aborting exceptions before writing
* stored fields. */
- private void fillStoredFields(int docID) throws IOException {
+ private void fillStoredFields(int docID) throws IOException, AbortingException {
while (lastStoredDocID < docID) {
startStoredFields();
finishStoredFields();
@@ -217,11 +217,7 @@ final class DefaultIndexingChain extends
@Override
public void abort() {
- try {
- // E.g. close any open files in the stored fields writer:
- storedFieldsWriter.abort();
- } catch (Throwable t) {
- }
+ IOUtils.closeWhileHandlingException(storedFieldsWriter);
try {
// E.g. close any open files in the term vectors writer:
@@ -257,36 +253,28 @@ final class DefaultIndexingChain extends
/** Calls StoredFieldsWriter.startDocument, aborting the
* segment if it hits any exception. */
- private void startStoredFields() throws IOException {
- boolean success = false;
+ private void startStoredFields() throws IOException, AbortingException {
try {
initStoredFieldsWriter();
storedFieldsWriter.startDocument();
- success = true;
- } finally {
- if (success == false) {
- docWriter.setAborting();
- }
+ } catch (Throwable th) {
+ throw new AbortingException(th);
}
lastStoredDocID++;
}
/** Calls StoredFieldsWriter.finishDocument, aborting the
* segment if it hits any exception. */
- private void finishStoredFields() throws IOException {
- boolean success = false;
+ private void finishStoredFields() throws IOException, AbortingException {
try {
storedFieldsWriter.finishDocument();
- success = true;
- } finally {
- if (success == false) {
- docWriter.setAborting();
- }
+ } catch (Throwable th) {
+ throw new AbortingException(th);
}
}
@Override
- public void processDocument(Term delTerm) throws IOException {
+ public void processDocument(Term delTerm) throws IOException, AbortingException {
// How many indexed field names we've seen (collapses
// multiple field instances by the same name):
@@ -306,12 +294,16 @@ final class DefaultIndexingChain extends
fillStoredFields(docState.docID);
startStoredFields();
+ boolean aborting = false;
try {
for (IndexableField field : docState.doc) {
fieldCount = processField(field, fieldGen, fieldCount, delTerm);
}
+ } catch (AbortingException ae) {
+ aborting = true;
+ throw ae;
} finally {
- if (docWriter.aborting == false) {
+ if (aborting == false) {
// Finish each indexed field name seen in the document:
for (int i=0;i<fieldCount;i++) {
fields[i].finish();
@@ -320,20 +312,16 @@ final class DefaultIndexingChain extends
}
}
- boolean success = false;
try {
termsHash.finishDocument();
- success = true;
- } finally {
- if (success == false) {
- // Must abort, on the possibility that on-disk term
- // vectors are now corrupt:
- docWriter.setAborting();
- }
+ } catch (Throwable th) {
+ // Must abort, on the possibility that on-disk term
+ // vectors are now corrupt:
+ throw new AbortingException(th);
}
}
- private int processField(IndexableField field, long fieldGen, int fieldCount, Term delTerm) throws IOException {
+ private int processField(IndexableField field, long fieldGen, int fieldCount, Term delTerm) throws IOException, AbortingException {
String fieldName = field.name();
IndexableFieldType fieldType = field.fieldType();
@@ -369,14 +357,10 @@ final class DefaultIndexingChain extends
fp = getOrAddField(fieldName, fieldType, false);
}
if (fieldType.stored()) {
- boolean success = false;
try {
storedFieldsWriter.writeField(fp.fieldInfo, field);
- success = true;
- } finally {
- if (!success) {
- docWriter.setAborting();
- }
+ } catch (Throwable th) {
+ throw new AbortingException(th);
}
}
}
@@ -598,7 +582,7 @@ final class DefaultIndexingChain extends
/** Inverts one field for one document; first is true
* if this is the first time we are seeing this field
* name in this document. */
- public void invert(IndexableField field, boolean first, Term delTerm) throws IOException {
+ public void invert(IndexableField field, boolean first, Term delTerm) throws IOException, AbortingException {
IndexableFieldType fieldType = field.fieldType();
if (first) {
// First time we're seeing this field (indexed) in
@@ -626,7 +610,6 @@ final class DefaultIndexingChain extends
* when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses,
* but rather a finally that takes note of the problem.
*/
- boolean aborting = false;
boolean succeededInProcessingField = false;
try (TokenStream stream = tokenStream = field.tokenStream(tokenStream)) {
// reset the TokenStream to the first token
@@ -678,9 +661,21 @@ final class DefaultIndexingChain extends
// internal state of the terms hash is now
// corrupt and should not be flushed to a
// new segment:
- aborting = true;
- termsHashPerField.add();
- aborting = false;
+ try {
+ termsHashPerField.add();
+ } catch (MaxBytesLengthExceededException e) {
+ byte[] prefix = new byte[30];
+ BytesRef bigTerm = invertState.termAttribute.getBytesRef();
+ System.arraycopy(bigTerm.bytes, bigTerm.offset, prefix, 0, 30);
+ String msg = "Document contains at least one immense term in field=\"" + fieldInfo.name + "\" (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + Arrays.toString(prefix) + "...', original message: " + e.getMessage();
+ if (docState.infoStream.isEnabled("IW")) {
+ docState.infoStream.message("IW", "ERROR: " + msg);
+ }
+ // Document will be deleted above:
+ throw new IllegalArgumentException(msg, e);
+ } catch (Throwable th) {
+ throw new AbortingException(th);
+ }
// maybe low-schema should know "isUnique"?
@@ -710,22 +705,7 @@ final class DefaultIndexingChain extends
/* if there is an exception coming through, we won't set this to true here:*/
succeededInProcessingField = true;
- } catch (MaxBytesLengthExceededException e) {
- aborting = false;
- byte[] prefix = new byte[30];
- BytesRef bigTerm = invertState.termAttribute.getBytesRef();
- System.arraycopy(bigTerm.bytes, bigTerm.offset, prefix, 0, 30);
- String msg = "Document contains at least one immense term in field=\"" + fieldInfo.name + "\" (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + Arrays.toString(prefix) + "...', original message: " + e.getMessage();
- if (docState.infoStream.isEnabled("IW")) {
- docState.infoStream.message("IW", "ERROR: " + msg);
- }
- // Document will be deleted above:
- throw new IllegalArgumentException(msg, e);
} finally {
- if (succeededInProcessingField == false && aborting) {
- docState.docWriter.setAborting();
- }
-
if (!succeededInProcessingField && docState.infoStream.isEnabled("DW")) {
docState.infoStream.message("DW", "An exception was thrown while processing field " + fieldInfo.name);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java Sun Dec 7 11:37:32 2014
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
abstract class DocConsumer {
- abstract void processDocument(Term delTerm) throws IOException;
- abstract void flush(final SegmentWriteState state) throws IOException;
+ abstract void processDocument(Term delTerm) throws IOException, AbortingException;
+ abstract void flush(final SegmentWriteState state) throws IOException, AbortingException;
abstract void abort();
}