You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/08 12:32:23 UTC
svn commit: r1637544 [1/6] - in /lucene/dev/branches/lucene6005: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/perfield/
lucene/core/src/java/org/apache/lucene/document/
lucene/core/src/java/org/apache/lucene/index/ lucene/core/...
Author: mikemccand
Date: Sat Nov 8 11:32:18 2014
New Revision: 1637544
URL: http://svn.apache.org/r1637544
Log:
LUCENE-6005: merge trunk
Added:
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/TooComplexToDeterminizeException.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/automaton/TooComplexToDeterminizeException.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestBitUtil.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestBitUtil.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreExplanations.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreExplanations.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
- copied unchanged from r1637540, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/MapSerializable.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/MapSerializable.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java
- copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java
Removed:
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BroadWord.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestExplanations.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java
Modified:
lucene/dev/branches/lucene6005/ (props changed)
lucene/dev/branches/lucene6005/lucene/ (props changed)
lucene/dev/branches/lucene6005/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/lucene/core/ (props changed)
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDecoder.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestGraphTokenizers.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestComplexExplanations.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestFieldValueFilter.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java
lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
lucene/dev/branches/lucene6005/lucene/facet/ (props changed)
lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.java
lucene/dev/branches/lucene6005/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
lucene/dev/branches/lucene6005/lucene/misc/ (props changed)
lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java
lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
lucene/dev/branches/lucene6005/lucene/queries/ (props changed)
lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java
lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
lucene/dev/branches/lucene6005/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene6005/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
lucene/dev/branches/lucene6005/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java
lucene/dev/branches/lucene6005/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
lucene/dev/branches/lucene6005/lucene/spatial/ (props changed)
lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java
lucene/dev/branches/lucene6005/lucene/suggest/ (props changed)
lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
lucene/dev/branches/lucene6005/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java
lucene/dev/branches/lucene6005/solr/ (props changed)
lucene/dev/branches/lucene6005/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6005/solr/bin/ (props changed)
lucene/dev/branches/lucene6005/solr/bin/solr
lucene/dev/branches/lucene6005/solr/bin/solr.cmd
lucene/dev/branches/lucene6005/solr/core/ (props changed)
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/SolrConfigRestApi.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/BBoxField.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/CacheConfig.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java
lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/PivotListEntry.java
lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestInitParams.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/TestRestManager.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java
lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java
lucene/dev/branches/lucene6005/solr/solrj/ (props changed)
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
lucene/dev/branches/lucene6005/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/branches/lucene6005/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
lucene/dev/branches/lucene6005/solr/test-framework/ (props changed)
lucene/dev/branches/lucene6005/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
Modified: lucene/dev/branches/lucene6005/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/CHANGES.txt?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6005/lucene/CHANGES.txt Sat Nov 8 11:32:18 2014
@@ -201,6 +201,24 @@ Bug Fixes
org.apache.lucene.index (Simon Willnauer, Robert Muir, Mike
McCandless)
+* LUCENE-6041: Remove sugar methods FieldInfo.isIndexed and
+ FieldInfo.hasDocValues. (Robert Muir, Mike McCandless)
+
+* LUCENE-6042: CustomScoreQuery explain was incorrect in some cases,
+ such as when nested inside a boolean query. (Denis Lantsman via Robert Muir)
+
+* LUCENE-6046: Add maxDeterminizedStates safety to determinize (which has
+ an exponential worst case) so that if it would create too many states, it
+ now throws an exception instead of exhausting CPU/RAM. (Nik
+ Everett via Mike McCandless)
+
+* LUCENE-6054: Allow repeating the empty automaton (Nik Everett via
+ Mike McCandless)
+
+* LUCENE-6049: Don't throw cryptic exception writing a segment when
+ the only docs in it had fields that hit non-aborting exceptions
+ during indexing but also had doc values. (Mike McCandless)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -253,6 +271,9 @@ Optimizations
* LUCENE-6030: Add norms patched compression for a small number of common values
(Ryan Ernst)
+* LUCENE-6040: Speed up EliasFanoDocIdSet through broadword bit selection.
+ (Paul Elschot)
+
Build
* LUCENE-5909: Smoke tester now has better command line parsing and
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Sat Nov 8 11:32:18 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@@ -251,7 +252,7 @@ public abstract class PerFieldDocValuesF
try {
// Read field name -> format name
for (FieldInfo fi : readState.fieldInfos) {
- if (fi.hasDocValues()) {
+ if (fi.getDocValuesType() != DocValuesType.NONE) {
final String fieldName = fi.name;
final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY);
if (formatName != null) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Sat Nov 8 11:32:18 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
@@ -242,7 +243,7 @@ public abstract class PerFieldPostingsFo
try {
// Read field name -> format name
for (FieldInfo fi : readState.fieldInfos) {
- if (fi.isIndexed()) {
+ if (fi.getIndexOptions() != IndexOptions.NONE) {
final String fieldName = fi.name;
final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY);
if (formatName != null) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java Sat Nov 8 11:32:18 2014
@@ -46,7 +46,7 @@ public class BinaryDocValuesField extend
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.BINARY);
+ TYPE.setDocValuesType(DocValuesType.BINARY);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java Sat Nov 8 11:32:18 2014
@@ -270,7 +270,7 @@ public final class Document implements I
return new FilterIterator<IndexableField, IndexableField>(fields.iterator()) {
@Override
protected boolean predicateFunction(IndexableField field) {
- return field.fieldType().stored() || field.fieldType().docValueType() != DocValuesType.NONE;
+ return field.fieldType().stored() || field.fieldType().docValuesType() != DocValuesType.NONE;
}
};
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Nov 8 11:32:18 2014
@@ -54,7 +54,7 @@ public class FieldType implements Indexa
private NumericType numericType;
private boolean frozen;
private int numericPrecisionStep = NumericUtils.PRECISION_STEP_DEFAULT;
- private DocValuesType docValueType = DocValuesType.NONE;
+ private DocValuesType docValuesType = DocValuesType.NONE;
/**
* Create a new mutable FieldType with all of the properties from <code>ref</code>
@@ -68,7 +68,7 @@ public class FieldType implements Indexa
this.storeTermVectorPayloads = ref.storeTermVectorPayloads();
this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions();
- this.docValueType = ref.docValueType();
+ this.docValuesType = ref.docValuesType();
this.numericType = ref.numericType();
// Do not copy frozen!
}
@@ -377,12 +377,12 @@ public class FieldType implements Indexa
result.append(numericPrecisionStep);
}
}
- if (docValueType != DocValuesType.NONE) {
+ if (docValuesType != DocValuesType.NONE) {
if (result.length() > 0) {
result.append(",");
}
- result.append("docValueType=");
- result.append(docValueType);
+ result.append("docValuesType=");
+ result.append(docValuesType);
}
return result.toString();
@@ -394,11 +394,11 @@ public class FieldType implements Indexa
* {@inheritDoc}
* <p>
* The default is <code>null</code> (no docValues)
- * @see #setDocValueType(DocValuesType)
+ * @see #setDocValuesType(DocValuesType)
*/
@Override
- public DocValuesType docValueType() {
- return docValueType;
+ public DocValuesType docValuesType() {
+ return docValuesType;
}
/**
@@ -406,21 +406,21 @@ public class FieldType implements Indexa
* @param type DocValues type, or null if no DocValues should be stored.
* @throws IllegalStateException if this FieldType is frozen against
* future modifications.
- * @see #docValueType()
+ * @see #docValuesType()
*/
- public void setDocValueType(DocValuesType type) {
+ public void setDocValuesType(DocValuesType type) {
checkIfFrozen();
if (type == null) {
throw new NullPointerException("DocValuesType cannot be null");
}
- docValueType = type;
+ docValuesType = type;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((docValueType == null) ? 0 : docValueType.hashCode());
+ result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());
result = prime * result + indexOptions.hashCode();
result = prime * result + numericPrecisionStep;
result = prime * result + ((numericType == null) ? 0 : numericType.hashCode());
@@ -440,7 +440,7 @@ public class FieldType implements Indexa
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
FieldType other = (FieldType) obj;
- if (docValueType != other.docValueType) return false;
+ if (docValuesType != other.docValuesType) return false;
if (indexOptions != other.indexOptions) return false;
if (numericPrecisionStep != other.numericPrecisionStep) return false;
if (numericType != other.numericType) return false;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Sat Nov 8 11:32:18 2014
@@ -748,7 +748,7 @@ public class FieldTypes {
}
@Override
- public DocValuesType docValueType() {
+ public DocValuesType docValuesType() {
return docValuesType;
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java Sat Nov 8 11:32:18 2014
@@ -40,7 +40,7 @@ public class NumericDocValuesField exten
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.NUMERIC);
+ TYPE.setDocValuesType(DocValuesType.NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java Sat Nov 8 11:32:18 2014
@@ -43,7 +43,7 @@ public class SortedDocValuesField extend
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED);
+ TYPE.setDocValuesType(DocValuesType.SORTED);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java Sat Nov 8 11:32:18 2014
@@ -50,7 +50,7 @@ public class SortedNumericDocValuesField
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED_NUMERIC);
+ TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java Sat Nov 8 11:32:18 2014
@@ -44,7 +44,7 @@ public class SortedSetDocValuesField ext
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED_SET);
+ TYPE.setDocValuesType(DocValuesType.SORTED_SET);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Nov 8 11:32:18 2014
@@ -57,7 +57,6 @@ import org.apache.lucene.util.StringHelp
import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-
/**
* Basic tool and API to check the health of an index and
* write a new segments file that removes reference to
@@ -1127,7 +1126,7 @@ public class CheckIndex implements Close
if (fieldInfo == null) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field);
}
- if (!fieldInfo.isIndexed()) {
+ if (fieldInfo.getIndexOptions() == IndexOptions.NONE) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field);
}
@@ -1769,7 +1768,7 @@ public class CheckIndex implements Close
infoStream.print(" test: docvalues...........");
}
for (FieldInfo fieldInfo : reader.getFieldInfos()) {
- if (fieldInfo.hasDocValues()) {
+ if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
status.totalValueFields++;
checkDocValues(fieldInfo, reader, infoStream, status);
} else {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Nov 8 11:32:18 2014
@@ -129,7 +129,7 @@ final class DefaultIndexingChain extends
PerField perField = fieldHash[i];
while (perField != null) {
if (perField.docValuesWriter != null) {
- if (perField.fieldInfo.hasDocValues() == false) {
+ if (perField.fieldInfo.getDocValuesType() == DocValuesType.NONE) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no docValues but wrote them");
}
@@ -142,7 +142,7 @@ final class DefaultIndexingChain extends
perField.docValuesWriter.finish(docCount);
perField.docValuesWriter.flush(state, dvConsumer);
perField.docValuesWriter = null;
- } else if (perField.fieldInfo.hasDocValues()) {
+ } else if (perField.fieldInfo.getDocValuesType() != DocValuesType.NONE) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has docValues but did not write them");
}
@@ -199,7 +199,7 @@ final class DefaultIndexingChain extends
// we must check the final value of omitNorms for the fieldinfo: it could have
// changed for this field since the first time we added it.
- if (fi.omitsNorms() == false && fi.isIndexed()) {
+ if (fi.omitsNorms() == false && fi.getIndexOptions() != IndexOptions.NONE) {
assert perField.norms != null: "field=" + fi.name;
perField.norms.finish(state.segmentInfo.getDocCount());
perField.norms.flush(state, normsConsumer);
@@ -382,7 +382,7 @@ final class DefaultIndexingChain extends
}
}
- DocValuesType dvType = fieldType.docValueType();
+ DocValuesType dvType = fieldType.docValuesType();
if (dvType == null) {
throw new NullPointerException("docValueType cannot be null (field: \"" + fieldName + "\")");
}
@@ -419,9 +419,7 @@ final class DefaultIndexingChain extends
* value */
private void indexDocValue(PerField fp, DocValuesType dvType, IndexableField field) throws IOException {
- boolean hasDocValues = fp.fieldInfo.hasDocValues();
-
- if (hasDocValues == false) {
+ if (fp.fieldInfo.getDocValuesType() == DocValuesType.NONE) {
// This will throw an exc if the caller tried to
// change the DV type for the field:
fieldInfos.globalFieldNumbers.setDocValuesType(fp.fieldInfo.number, fp.fieldInfo.name, dvType);
@@ -499,7 +497,11 @@ final class DefaultIndexingChain extends
if (fp == null) {
// First time we are seeing this field in this segment
- FieldInfo fi = fieldInfos.addOrUpdate(name, fieldType);
+ FieldInfo fi = fieldInfos.getOrAdd(name);
+ // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
+ // IndexOptions to decide what arrays it must create). Then, we also must set it in
+ // PerField.invert to allow for later downgrading of the index options:
+ fi.setIndexOptions(fieldType.indexOptions());
fp = new PerField(fi, invert);
fp.next = fieldHash[hashPos];
@@ -518,7 +520,10 @@ final class DefaultIndexingChain extends
}
} else {
- fp.fieldInfo.update(fieldType);
+ // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
+ // IndexOptions to decide what arrays it must create). Then, we also must set it in
+ // PerField.invert to allow for later downgrading of the index options:
+ fp.fieldInfo.setIndexOptions(fieldType.indexOptions());
// NOTE: messy, but we must do this in case field was first seen w/o being
// indexed, and now is seen again, this time being indexed:
@@ -557,6 +562,8 @@ final class DefaultIndexingChain extends
// reused
TokenStream tokenStream;
+ IndexOptions indexOptions;
+
public PerField(FieldInfo fieldInfo, boolean invert) {
this.fieldInfo = fieldInfo;
similarity = docState.similarity;
@@ -605,10 +612,17 @@ final class DefaultIndexingChain extends
IndexableFieldType fieldType = field.fieldType();
+ IndexOptions indexOptions = fieldType.indexOptions();
+ fieldInfo.setIndexOptions(indexOptions);
+
+ if (fieldType.omitNorms()) {
+ fieldInfo.setOmitsNorms();
+ }
+
// only bother checking offsets if something will consume them.
// nocommit can't we do this todo now?
// TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
- final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+ final boolean checkOffsets = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
/*
* To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Sat Nov 8 11:32:18 2014
@@ -108,10 +108,6 @@ public final class FieldInfo {
return true;
}
- void update(IndexableFieldType ft) {
- update(false, ft.omitNorms(), false, ft.indexOptions());
- }
-
// should only be called by FieldInfos#addOrUpdate
void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
if (indexOptions == null) {
@@ -158,14 +154,24 @@ public final class FieldInfo {
public IndexOptions getIndexOptions() {
return indexOptions;
}
-
- /**
- * Returns true if this field has any docValues.
- */
- public boolean hasDocValues() {
- return docValuesType != DocValuesType.NONE;
- }
+ /** Record the {@link IndexOptions} to use with this field. */
+ public void setIndexOptions(IndexOptions newIndexOptions) {
+ if (indexOptions != newIndexOptions) {
+ if (indexOptions == IndexOptions.NONE) {
+ indexOptions = newIndexOptions;
+ } else if (newIndexOptions != IndexOptions.NONE) {
+ // downgrade
+ indexOptions = indexOptions.compareTo(newIndexOptions) < 0 ? indexOptions : newIndexOptions;
+ }
+ }
+
+ if (indexOptions == IndexOptions.NONE || indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ // cannot store payloads if we don't store positions:
+ storePayloads = false;
+ }
+ }
+
/**
* Returns {@link DocValuesType} of the docValues; this is
* {@code DocValuesType.NONE} if the field has no docvalues.
@@ -206,19 +212,20 @@ public final class FieldInfo {
public boolean omitsNorms() {
return omitNorms;
}
+
+ /** Omit norms for this field. */
+ public void setOmitsNorms() {
+ if (indexOptions == IndexOptions.NONE) {
+ throw new IllegalStateException("cannot omit norms: this field is not indexed");
+ }
+ omitNorms = true;
+ }
/**
* Returns true if this field actually has any norms.
*/
public boolean hasNorms() {
- return isIndexed() && omitNorms == false;
- }
-
- /**
- * Returns true if this field is indexed ({@link #getIndexOptions} is not IndexOptions.NONE).
- */
- public boolean isIndexed() {
- return indexOptions != IndexOptions.NONE;
+ return indexOptions != IndexOptions.NONE && omitNorms == false;
}
/**
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Sat Nov 8 11:32:18 2014
@@ -68,11 +68,11 @@ public class FieldInfos implements Itera
}
hasVectors |= info.hasVectors();
- hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS;
- hasOffsets |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ hasProx |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ hasFreq |= info.getIndexOptions() != IndexOptions.DOCS;
+ hasOffsets |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
hasNorms |= info.hasNorms();
- hasDocValues |= info.hasDocValues();
+ hasDocValues |= info.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= info.hasPayloads();
}
@@ -276,21 +276,26 @@ public class FieldInfos implements Itera
add(fieldInfo);
}
}
-
- /** NOTE: this method does not carry over termVector
- * the indexer chain must set these fields when they
- * succeed in consuming the document, nor the DocValuesType */
- public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
- // TODO: really, indexer shouldn't even call this
- // method (it's only called from DocFieldProcessor);
- // rather, each component in the chain should update
- // what it "owns". EG fieldType.indexOptions() should
- // be updated by maybe FreqProxTermsWriterPerField:
- return addOrUpdateInternal(name, -1, false,
- fieldType.omitNorms(), false,
- fieldType.indexOptions(), DocValuesType.NONE);
- }
+ /** Create a new field, or return existing one. */
+ public FieldInfo getOrAdd(String name) {
+ FieldInfo fi = fieldInfo(name);
+ if (fi == null) {
+ // This field wasn't yet added to this in-RAM
+ // segment's FieldInfo, so now we get a global
+ // number for this field. If the field was seen
+ // before then we'll get the same name and number,
+ // else we'll allocate a new one:
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE);
+ fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, null);
+ assert !byName.containsKey(fi.name);
+ globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
+ byName.put(fi.name, fi);
+ }
+
+ return fi;
+ }
+
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
@@ -314,10 +319,10 @@ public class FieldInfos implements Itera
if (docValues != DocValuesType.NONE) {
// Only pay the synchronization cost if fi does not already have a DVType
- boolean updateGlobal = !fi.hasDocValues();
+ boolean updateGlobal = fi.getDocValuesType() == DocValuesType.NONE;
if (updateGlobal) {
// Must also update docValuesType map so it's
- // aware of this field's DocValueType. This will throw IllegalArgumentException if
+ // aware of this field's DocValuesType. This will throw IllegalArgumentException if
// an illegal type change was attempted.
globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Sat Nov 8 11:32:18 2014
@@ -89,7 +89,7 @@ final class FreqProxTermsWriter extends
final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f;
if (perField.bytesHash.size() > 0) {
perField.sortPostings();
- assert perField.fieldInfo.isIndexed();
+ assert perField.fieldInfo.getIndexOptions() != IndexOptions.NONE;
allFields.add(perField);
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov 8 11:32:18 2014
@@ -1532,7 +1532,7 @@ public class IndexWriter implements Clos
DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length];
for (int i = 0; i < updates.length; i++) {
final Field f = updates[i];
- final DocValuesType dvType = f.fieldType().docValueType();
+ final DocValuesType dvType = f.fieldType().docValuesType();
if (dvType == null) {
throw new NullPointerException("DocValuesType cannot be null (field: \"" + f.name() + "\")");
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Sat Nov 8 11:32:18 2014
@@ -84,5 +84,5 @@ public interface IndexableFieldType {
* DocValues {@link DocValuesType}: how the field's value will be indexed
* into docValues.
*/
- public DocValuesType docValueType();
+ public DocValuesType docValuesType();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java Sat Nov 8 11:32:18 2014
@@ -19,11 +19,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Collection;
-import java.util.HashSet;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.util.Bits;
@@ -271,7 +271,7 @@ public final class MultiFields extends F
public static Collection<String> getIndexedFields(IndexReader reader) {
final Collection<String> fields = new HashSet<>();
for(final FieldInfo fieldInfo : getMergedFieldInfos(reader)) {
- if (fieldInfo.isIndexed()) {
+ if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
fields.add(fieldInfo.name);
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Sat Nov 8 11:32:18 2014
@@ -497,12 +497,12 @@ class ReadersAndUpdates {
}
// create new fields or update existing ones to have NumericDV type
for (String f : dvUpdates.numericDVUpdates.keySet()) {
- FieldInfo fieldInfo = builder.addOrUpdate(f, NumericDocValuesField.TYPE);
+ FieldInfo fieldInfo = builder.getOrAdd(f);
fieldInfo.setDocValuesType(DocValuesType.NUMERIC);
}
// create new fields or update existing ones to have BinaryDV type
for (String f : dvUpdates.binaryDVUpdates.keySet()) {
- FieldInfo fieldInfo = builder.addOrUpdate(f, BinaryDocValuesField.TYPE);
+ FieldInfo fieldInfo = builder.getOrAdd(f);
fieldInfo.setDocValuesType(DocValuesType.BINARY);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java Sat Nov 8 11:32:18 2014
@@ -53,7 +53,7 @@ class SegmentDocValuesProducer extends D
try {
DocValuesProducer baseProducer = null;
for (FieldInfo fi : fieldInfos) {
- if (!fi.hasDocValues()) {
+ if (fi.getDocValuesType() == DocValuesType.NONE) {
continue;
}
long docValuesGen = fi.getDocValuesGen();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java Sat Nov 8 11:32:18 2014
@@ -44,7 +44,7 @@ public class SimpleMergedSegmentWarmer e
int docValuesCount = 0;
int normsCount = 0;
for (FieldInfo info : reader.getFieldInfos()) {
- if (info.isIndexed()) {
+ if (info.getIndexOptions() != IndexOptions.NONE) {
reader.terms(info.name);
indexedCount++;
@@ -54,7 +54,7 @@ public class SimpleMergedSegmentWarmer e
}
}
- if (info.hasDocValues()) {
+ if (info.getDocValuesType() != DocValuesType.NONE) {
switch(info.getDocValuesType()) {
case NUMERIC:
reader.getNumericDocValues(info.name);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Sat Nov 8 11:32:18 2014
@@ -86,7 +86,7 @@ abstract class TermsHash {
}
}
- abstract TermsHashPerField addField(FieldInvertState fieldInvertState, final FieldInfo fieldInfo);
+ abstract TermsHashPerField addField(FieldInvertState fieldInvertState, FieldInfo fieldInfo);
void finishDocument() throws IOException {
if (nextTermsHash != null) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java Sat Nov 8 11:32:18 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.util.AttributeS
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
/**
* A {@link Query} that will match terms against a finite-state machine.
@@ -61,10 +62,26 @@ public class AutomatonQuery extends Mult
* match.
*/
public AutomatonQuery(final Term term, Automaton automaton) {
+ this(term, automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+ }
+
+ /**
+ * Create a new AutomatonQuery from an {@link Automaton}.
+ *
+ * @param term Term containing field and possibly some pattern structure. The
+ * term text is ignored.
+ * @param automaton Automaton to run, terms that are accepted are considered a
+ * match.
+ * @param maxDeterminizedStates maximum number of states in the resulting
+ * automata. If the automata would need more than this many states
+ * TooComplextToDeterminizeException is thrown. Higher number require more
+ * space but can process more complex automata.
+ */
+ public AutomatonQuery(final Term term, Automaton automaton, int maxDeterminizedStates) {
super(term.field());
this.term = term;
this.automaton = automaton;
- this.compiled = new CompiledAutomaton(automaton);
+ this.compiled = new CompiledAutomaton(automaton, null, true, maxDeterminizedStates);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java Sat Nov 8 11:32:18 2014
@@ -20,6 +20,8 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Bits.MatchAllBits;
import org.apache.lucene.util.Bits.MatchNoBits;
@@ -93,10 +95,10 @@ public class FieldValueFilter extends Fi
if (docsWithField instanceof MatchNoBits) {
return null;
}
- if (docsWithField instanceof DocIdSet) {
+ if (docsWithField instanceof BitSet) {
// UweSays: this is always the case for our current impl - but who knows
// :-)
- return BitsFilteredDocIdSet.wrap((DocIdSet) docsWithField, acceptDocs);
+ return BitsFilteredDocIdSet.wrap(new BitDocIdSet((BitSet) docsWithField), acceptDocs);
}
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java Sat Nov 8 11:32:18 2014
@@ -1,11 +1,5 @@
package org.apache.lucene.search;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.util.ToStringUtils;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.AutomatonProvider;
-import org.apache.lucene.util.automaton.RegExp;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -23,6 +17,13 @@ import org.apache.lucene.util.automaton.
* limitations under the License.
*/
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.AutomatonProvider;
+import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.RegExp;
+
/**
* A fast regular expression query based on the
* {@link org.apache.lucene.util.automaton} package.
@@ -75,18 +76,38 @@ public class RegexpQuery extends Automat
* @param flags optional RegExp features from {@link RegExp}
*/
public RegexpQuery(Term term, int flags) {
- this(term, flags, defaultProvider);
+ this(term, flags, defaultProvider,
+ Operations.DEFAULT_MAX_DETERMINIZED_STATES);
}
-
+
+ /**
+ * Constructs a query for terms matching <code>term</code>.
+ *
+ * @param term regular expression.
+ * @param flags optional RegExp features from {@link RegExp}
+ * @param maxDeterminizedStates maximum number of states that compiling the
+ * automaton for the regexp can result in. Set higher to allow more complex
+ * queries and lower to prevent memory exhaustion.
+ */
+ public RegexpQuery(Term term, int flags, int maxDeterminizedStates) {
+ this(term, flags, defaultProvider, maxDeterminizedStates);
+ }
+
/**
* Constructs a query for terms matching <code>term</code>.
*
* @param term regular expression.
* @param flags optional RegExp features from {@link RegExp}
* @param provider custom AutomatonProvider for named automata
+ * @param maxDeterminizedStates maximum number of states that compiling the
+ * automaton for the regexp can result in. Set higher to allow more complex
+ * queries and lower to prevent memory exhaustion.
*/
- public RegexpQuery(Term term, int flags, AutomatonProvider provider) {
- super(term, new RegExp(term.text(), flags).toAutomaton(provider));
+ public RegexpQuery(Term term, int flags, AutomatonProvider provider,
+ int maxDeterminizedStates) {
+ super(term,
+ new RegExp(term.text(), flags).toAutomaton(
+ provider, maxDeterminizedStates), maxDeterminizedStates);
}
/** Prints a user-readable version of this query. */
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java Sat Nov 8 11:32:18 2014
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.automaton.Automata;
-import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
/** Implements the wildcard search query. Supported wildcards are <code>*</code>, which
* matches any character sequence (including the empty one), and <code>?</code>,
@@ -58,6 +58,17 @@ public class WildcardQuery extends Autom
}
/**
+ * Constructs a query for terms matching <code>term</code>.
+ * @param maxDeterminizedStates maximum number of states in the resulting
+ * automata. If the automata would need more than this many states
+ * TooComplextToDeterminizeException is thrown. Higher number require more
+ * space but can process more complex automata.
+ */
+ public WildcardQuery(Term term, int maxDeterminizedStates) {
+ super(term, toAutomaton(term), maxDeterminizedStates);
+ }
+
+ /**
* Convert Lucene wildcard syntax into an automaton.
* @lucene.internal
*/
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java Sat Nov 8 11:32:18 2014
@@ -212,4 +212,64 @@ public final class BitUtil {
return ((l >>> 1) ^ -(l & 1));
}
+
+ /** Select a 1-bit from a long. See also LUCENE-6040.
+ * @return The index of the r-th 1 bit in x. This bit must exist.
+ */
+ public static int select(long x, int r) {
+ long s = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1); // pairwise bitsums
+ s = (s & 0x3333333333333333L) + ((s >>> 2) & 0x3333333333333333L); // nibblewise bitsums
+ s = ((s + (s >>> 4)) & 0x0F0F0F0F0F0F0F0FL) * L8_L; // bytewise bitsums, cumulative
+
+ int b = (Long.numberOfTrailingZeros((s + psOverflow[r-1]) & (L8_L << 7)) >> 3) << 3; // bit position of byte with r-th 1 bit.
+ long l = r - (((s << 8) >>> b) & 0xFFL); // bit rank in byte at b
+
+ // Select bit l from byte (x >>> b):
+ int selectIndex = (int) (((x >>> b) & 0xFFL) | ((l-1) << 8));
+ int res = b + select256[selectIndex];
+ return res;
+ }
+
+ private final static long L8_L = 0x0101010101010101L;
+
+ private static final long[] psOverflow = new long[64];
+ static {
+ for (int s = 1; s <= 64; s++) {
+ psOverflow[s-1] = (128-s) * L8_L;
+ }
+ }
+
+ private static final byte[] select256 = new byte[8 * 256];
+ static {
+ for (int b = 0; b <= 0xFF; b++) {
+ for (int s = 1; s <= 8; s++) {
+ int byteIndex = b | ((s-1) << 8);
+ int bitIndex = selectNaive(b, s);
+ if (bitIndex < 0) {
+ bitIndex = 127; // positive as byte
+ }
+ assert bitIndex >= 0;
+ assert ((byte) bitIndex) >= 0; // non negative as byte, no need to mask the sign
+ select256[byteIndex] = (byte) bitIndex;
+ }
+ }
+ }
+
+ /**
+ * Naive implementation of {@link #select(long,int)}, using {@link Long#numberOfTrailingZeros} repetitively.
+ * Works relatively fast for low ranks.
+ * @return The index of the r-th 1 bit in x, or -1 if no such bit exists.
+ */
+ public static int selectNaive(long x, int r) {
+ assert r >= 1;
+ int s = -1;
+ while ((x != 0L) && (r > 0)) {
+ int ntz = Long.numberOfTrailingZeros(x);
+ x >>>= (ntz + 1);
+ s += (ntz + 1);
+ r -= 1;
+ }
+ int res = (r > 0) ? -1 : s;
+ return res;
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Sat Nov 8 11:32:18 2014
@@ -37,6 +37,7 @@ import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -215,7 +216,7 @@ public final class IOUtils {
* <p>
* Some of the files may be null, if so they are ignored.
*/
- public static void deleteFilesIgnoringExceptions(Iterable<? extends Path> files) {
+ public static void deleteFilesIgnoringExceptions(Collection<? extends Path> files) {
for (Path name : files) {
if (name != null) {
try {
@@ -249,7 +250,7 @@ public final class IOUtils {
*
* @param files files to delete
*/
- public static void deleteFilesIfExist(Iterable<? extends Path> files) throws IOException {
+ public static void deleteFilesIfExist(Collection<? extends Path> files) throws IOException {
Throwable th = null;
for (Path file : files) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java Sat Nov 8 11:32:18 2014
@@ -33,7 +33,7 @@ import java.io.IOException;
/**
* Automaton provider for <code>RegExp.</code>
- * {@link RegExp#toAutomaton(AutomatonProvider)}
+ * {@link RegExp#toAutomaton(AutomatonProvider,int)}
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java Sat Nov 8 11:32:18 2014
@@ -24,12 +24,12 @@ public class ByteRunAutomaton extends Ru
/** Converts incoming automaton to byte-based (UTF32ToUTF8) first */
public ByteRunAutomaton(Automaton a) {
- this(a, false);
+ this(a, false, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
}
/** expert: if isBinary is true, the input is already byte-based, else it's UTF-32 and we'll convert to UTF-8 */
- public ByteRunAutomaton(Automaton a, boolean isBinary) {
- super(isBinary ? a : new UTF32ToUTF8().convert(a), 256, true);
+ public ByteRunAutomaton(Automaton a, boolean isBinary, int maxDeterminizedStates) {
+ super(isBinary ? a : new UTF32ToUTF8().convert(a), 256, true, maxDeterminizedStates);
}
/**
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java Sat Nov 8 11:32:18 2014
@@ -21,10 +21,22 @@ package org.apache.lucene.util.automaton
* Automaton representation for matching char[].
*/
public class CharacterRunAutomaton extends RunAutomaton {
-
- /** Sole constructor. */
+ /**
+ * Construct with a default number of maxDeterminizedStates.
+ */
public CharacterRunAutomaton(Automaton a) {
- super(a, Character.MAX_CODE_POINT, false);
+ this(a, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+ }
+
+ /**
+ * Construct specifying maxDeterminizedStates.
+ * @param a Automaton to match
+ * @param maxDeterminizedStates maximum number of states that the automataon
+ * can have once determinized. If more states are required to determinize
+ * it then a TooComplexToDeterminizeException is thrown.
+ */
+ public CharacterRunAutomaton(Automaton a, int maxDeterminizedStates) {
+ super(a, Character.MAX_CODE_POINT, false, maxDeterminizedStates);
}
/**
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java Sat Nov 8 11:32:18 2014
@@ -21,9 +21,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.index.SingleTermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.SingleTermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -157,7 +157,8 @@ public class CompiledAutomaton {
}
}
sinkState = foundState;
- runAutomaton = new ByteRunAutomaton(automaton, true);
+ // It's safe to allow unlimited determinized here:
+ runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE);
type = AUTOMATON_TYPE.RANGE;
}
@@ -177,7 +178,8 @@ public class CompiledAutomaton {
automaton.addTransition(lastState, lastState, 0, 255);
sinkState = lastState;
automaton.finishState();
- runAutomaton = new ByteRunAutomaton(automaton, true);
+ // It's safe to allow unlimited determinized here:
+ runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE);
commonSuffixRef = null;
finite = false;
minInclusive = false;
@@ -190,7 +192,20 @@ public class CompiledAutomaton {
* possibly expensive operations to determine if the automaton is one
* the cases in {@link CompiledAutomaton.AUTOMATON_TYPE}. */
public CompiledAutomaton(Automaton automaton, Boolean finite, boolean simplify) {
+ this(automaton, finite, simplify, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+ }
+
+ /** Create this. If finite is null, we use {@link Operations#isFinite}
+ * to determine whether it is finite. If simplify is true, we run
+ * possibly expensive operations to determine if the automaton is one
+ * the cases in {@link CompiledAutomaton.AUTOMATON_TYPE}. If simplify
+ * requires determinizing the autaomaton then only maxDeterminizedStates
+ * will be created. Any more than that will cause a
+ * TooComplexToDeterminizeException.
+ */
+ public CompiledAutomaton(Automaton automaton, Boolean finite, boolean simplify,
+ int maxDeterminizedStates) {
if (automaton.getNumStates() == 0) {
automaton = new Automaton();
automaton.createState();
@@ -231,7 +246,7 @@ public class CompiledAutomaton {
return;
} else {
- automaton = Operations.determinize(automaton);
+ automaton = Operations.determinize(automaton, maxDeterminizedStates);
final String commonPrefix = Operations.getCommonPrefix(automaton);
final String singleton;
@@ -257,7 +272,7 @@ public class CompiledAutomaton {
return;
} else if (commonPrefix.length() > 0) {
Automaton other = Operations.concatenate(Automata.makeString(commonPrefix), Automata.makeAnyString());
- other = Operations.determinize(other);
+ other = Operations.determinize(other, maxDeterminizedStates);
assert Operations.hasDeadStates(other) == false;
if (Operations.sameLanguage(automaton, other)) {
// matches a constant prefix
@@ -276,7 +291,8 @@ public class CompiledAutomaton {
sinkState = lastState;
automaton.finishState();
this.automaton = automaton;
- runAutomaton = new ByteRunAutomaton(automaton, true);
+ // It's safe to allow unlimited determinized here:
+ runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE);
this.finite = false;
maxTerm = null;
minInclusive = false;
@@ -305,9 +321,9 @@ public class CompiledAutomaton {
if (this.finite) {
commonSuffixRef = null;
} else {
- commonSuffixRef = Operations.getCommonSuffixBytesRef(binary);
+ commonSuffixRef = Operations.getCommonSuffixBytesRef(binary, maxDeterminizedStates);
}
- runAutomaton = new ByteRunAutomaton(binary, true);
+ runAutomaton = new ByteRunAutomaton(binary, true, maxDeterminizedStates);
this.automaton = runAutomaton.automaton;
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java Sat Nov 8 11:32:18 2014
@@ -29,8 +29,8 @@
package org.apache.lucene.util.automaton;
-import java.util.BitSet;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.HashSet;
import java.util.LinkedList;
@@ -45,21 +45,17 @@ final public class MinimizationOperation
/**
* Minimizes (and determinizes if not already deterministic) the given
- * automaton.
- */
- public static Automaton minimize(Automaton a) {
- return minimizeHopcroft(a);
- }
-
- /**
- * Minimizes the given automaton using Hopcroft's algorithm.
+ * automaton using Hopcroft's algorighm.
+ * @param maxDeterminizedStates maximum number of states determinizing the
+ * automaton can result in. Set higher to allow more complex queries and
+ * lower to prevent memory exhaustion.
*/
- public static Automaton minimizeHopcroft(Automaton a) {
+ public static Automaton minimize(Automaton a, int maxDeterminizedStates) {
if (a.getNumStates() == 0 || (a.isAccept(0) == false && a.getNumTransitions(0) == 0)) {
// Fastmatch for common case
return new Automaton();
}
- a = Operations.determinize(a);
+ a = Operations.determinize(a, maxDeterminizedStates);
//a.writeDot("adet");
if (a.getNumTransitions(0) == 1) {
Transition t = new Transition();