You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/04 17:08:35 UTC
svn commit: r1629403 [1/4] - in /lucene/dev/branches/lucene5969: ./
dev-tools/ dev-tools/scripts/ lucene/ lucene/backward-codecs/
lucene/backward-codecs/src/test/org/apache/lucene/index/ lucene/core/
lucene/core/src/java/org/apache/lucene/document/ luc...
Author: rmuir
Date: Sat Oct 4 15:08:32 2014
New Revision: 1629403
URL: http://svn.apache.org/r1629403
Log:
merge trunk
Added:
lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-cfs.zip
- copied unchanged from r1629401, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-cfs.zip
lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-nocfs.zip
- copied unchanged from r1629401, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-nocfs.zip
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
- copied unchanged from r1629401, lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/SchemaZkVersionResource.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaZkVersionResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
lucene/dev/branches/lucene5969/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml
lucene/dev/branches/lucene5969/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog-with-delayingcomponent.xml
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog-with-delayingcomponent.xml
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/search/DelayingSearchComponent.java
- copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/DelayingSearchComponent.java
Removed:
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java
lucene/dev/branches/lucene5969/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java
Modified:
lucene/dev/branches/lucene5969/ (props changed)
lucene/dev/branches/lucene5969/dev-tools/ (props changed)
lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py
lucene/dev/branches/lucene5969/lucene/ (props changed)
lucene/dev/branches/lucene5969/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5969/lucene/backward-codecs/ (props changed)
lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/lucene5969/lucene/core/ (props changed)
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java
lucene/dev/branches/lucene5969/lucene/demo/ (props changed)
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/AssociationsFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/ExpressionAggregationFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/MultiCategoryListsFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java
lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java
lucene/dev/branches/lucene5969/lucene/memory/ (props changed)
lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/lucene5969/lucene/queries/ (props changed)
lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java
lucene/dev/branches/lucene5969/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
lucene/dev/branches/lucene5969/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
lucene/dev/branches/lucene5969/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
lucene/dev/branches/lucene5969/lucene/replicator/ (props changed)
lucene/dev/branches/lucene5969/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
lucene/dev/branches/lucene5969/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java
lucene/dev/branches/lucene5969/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/branches/lucene5969/solr/ (props changed)
lucene/dev/branches/lucene5969/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5969/solr/core/ (props changed)
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
lucene/dev/branches/lucene5969/solr/solrj/ (props changed)
lucene/dev/branches/lucene5969/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/lucene5969/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
lucene/dev/branches/lucene5969/solr/test-framework/ (props changed)
lucene/dev/branches/lucene5969/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Modified: lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py (original)
+++ lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py Sat Oct 4 15:08:32 2014
@@ -161,9 +161,11 @@ def checkClassSummaries(fullPath):
foundMethodDetail = False
lastMethodAnchor = None
+ lineCount = 0
for line in f.readlines():
m = reMethodDetail.search(line)
+ lineCount += 1
if m is not None:
foundMethodDetail = True
continue
@@ -212,6 +214,8 @@ def checkClassSummaries(fullPath):
if inThing:
if lineLower.find('</tr>') != -1:
if not hasDesc:
+ if lastItem is None:
+ raise RuntimeError('failed to locate javadoc item in %s, line %d? last line: %s' % (fullPath, lineCount, line.rstrip()))
missing.append((lastCaption, unEscapeURL(lastItem)))
inThing = False
continue
Modified: lucene/dev/branches/lucene5969/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/CHANGES.txt?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5969/lucene/CHANGES.txt Sat Oct 4 15:08:32 2014
@@ -70,6 +70,9 @@ New Features
exiting requests that take too long to enumerate over terms. (Anshum Gupta, Steve Rowe,
Robert Muir)
+* LUCENE-5911: Add MemoryIndex.freeze() to allow thread-safe searching over a
+ MemoryIndex. (Alan Woodward)
+
API Changes
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
@@ -133,42 +136,30 @@ API Changes
* LUCENE-5569: *AtomicReader/AtomicReaderContext have been renamed to *LeafReader/LeafReaderContext.
(Ryan Ernst)
+* LUCENE-5938: Removed MultiTermQuery.ConstantScoreAutoRewrite as
+ MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is usually better. (Adrien Grand)
+
+* LUCENE-5924: Rename CheckIndex -fix option to -exorcise. This option does not
+ actually fix the index, it just drops data. (Robert Muir)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
folder via security manager, and make test temp dirs absolute.
(Ryan Ernst, Dawid Weiss)
-* LUCENE-5904: Fixed a corruption case that can happen when 1)
- IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
- on startup, when a new IndexWriter is created, a virus checker is
- holding some of the previously written but unused files open and
- preventing deletion, 3) IndexWriter writes these files again during
- the course of indexing, then the files can later be deleted, causing
- corruption. This case was detected by adding evilness to
- MockDirectoryWrapper to have it simulate a virus checker holding a
- file open and preventing deletion (Robert Muir, Mike McCandless)
-
-* LUCENE-5916: Static scope test components should be consistent between
- tests (and test iterations). Fix for FaultyIndexInput in particular.
- (Dawid Weiss)
-
-* LUCENE-5934: Fix backwards compatibility for 4.0 indexes.
- (Ian Lea, Uwe Schindler, Robert Muir, Ryan Ernst)
-
-* LUCENE-5958: Don't let exceptions during checkpoint corrupt the index.
- Refactor existing OOM handling too, so you don't need to handle OOM special
- for every IndexWriter method: instead such disasters will cause IW to close itself
- defensively. (Robert Muir, Mike McCandless)
-
-* LUCENE-5952: Improve error messages when version cannot be parsed;
- don't check for too old or too new major version (it's too low level
- to enforce here); use simple string tokenizer. (Ryan Ernst, Uwe Schindler,
- Robert Muir, Mike McCandless)
-
* LUCENE-5948: RateLimiter now fully inits itself on init. (Varun
Thacker via Mike McCandless)
+* LUCENE-5977: Fix tokenstream safety checks in IndexWriter to properly
+ work across multi-valued fields. Previously some cases across multi-valued
+ fields would happily create a corrupt index. (Dawid Weiss, Robert Muir)
+
+* LUCENE-5981: CheckIndex obtains write.lock, since with some parameters it
+ may modify the index, and to prevent false corruption reports, as it does
+ not have the regular "spinlock" of DirectoryReader.open. It now implements
+ Closeable and you must close it to release the lock. (Mike McCandless, Robert Muir)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -176,13 +167,6 @@ Documentation
Tests
-* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
- (Ryan Ernst)
-
-* LUCENE-5939: Regenerate old backcompat indexes to ensure they were built with
- the exact release
- (Ryan Ernst, Uwe Schindler)
-
* LUCENE-5957: Add option for tests to not randomize codec
(Ryan Ernst)
@@ -205,6 +189,10 @@ Optimizations
* LUCENE-5963: Reduce memory allocations in
AnalyzingSuggester. (Markus Heiden via Mike McCandless)
+* LUCENE-5938: MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is now faster on
+ queries that match few documents by using a sparse bit set implementation.
+ (Adrien Grand)
+
Build
* LUCENE-5909: Smoke tester now has better command line parsing and
@@ -225,6 +213,50 @@ Other
* LUCENE-5915: Remove Pulsing postings format. (Robert Muir)
+======================= Lucene 4.10.1 ======================
+
+Bug fixes
+
+* LUCENE-5934: Fix backwards compatibility for 4.0 indexes.
+ (Ian Lea, Uwe Schindler, Robert Muir, Ryan Ernst)
+
+* LUCENE-5939: Regenerate old backcompat indexes to ensure they were built with
+ the exact release
+ (Ryan Ernst, Uwe Schindler)
+
+* LUCENE-5952: Improve error messages when version cannot be parsed;
+ don't check for too old or too new major version (it's too low level
+ to enforce here); use simple string tokenizer. (Ryan Ernst, Uwe Schindler,
+ Robert Muir, Mike McCandless)
+
+* LUCENE-5958: Don't let exceptions during checkpoint corrupt the index.
+ Refactor existing OOM handling too, so you don't need to handle OOM special
+ for every IndexWriter method: instead such disasters will cause IW to close itself
+ defensively. (Robert Muir, Mike McCandless)
+
+* LUCENE-5904: Fixed a corruption case that can happen when 1)
+ IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
+ on startup, when a new IndexWriter is created, a virus checker is
+ holding some of the previously written but unused files open and
+ preventing deletion, 3) IndexWriter writes these files again during
+ the course of indexing, then the files can later be deleted, causing
+ corruption. This case was detected by adding evilness to
+ MockDirectoryWrapper to have it simulate a virus checker holding a
+ file open and preventing deletion (Robert Muir, Mike McCandless)
+
+* LUCENE-5916: Static scope test components should be consistent between
+ tests (and test iterations). Fix for FaultyIndexInput in particular.
+ (Dawid Weiss)
+
+* LUCENE-5975: Fix reading of 3.0-3.3 indexes, where bugs in these old
+ index formats would result in CorruptIndexException "did not read all
+ bytes from file" when reading the deleted docs file. (Patrick Mi, Robert MUir)
+
+Tests
+
+* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
+ (Ryan Ernst)
+
======================= Lucene 4.10.0 ======================
New Features
Modified: lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat Oct 4 15:08:32 2014
@@ -250,7 +250,9 @@ public class TestBackwardsCompatibility
"4.9.1-cfs",
"4.9.1-nocfs",
"4.10.0-cfs",
- "4.10.0-nocfs"
+ "4.10.0-nocfs",
+ "4.10.1-cfs",
+ "4.10.1-nocfs"
};
final String[] unsupportedNames = {
@@ -514,6 +516,7 @@ public class TestBackwardsCompatibility
CheckIndex.Status indexStatus = checker.checkIndex();
assertFalse(indexStatus.clean);
assertTrue(bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName()));
+ checker.close();
dir.close();
IOUtils.rm(oldIndexDir);
@@ -749,12 +752,6 @@ public class TestBackwardsCompatibility
reader.close();
}
- private int compare(String name, String v) {
- int v0 = Integer.parseInt(name.substring(0, 2));
- int v1 = Integer.parseInt(v);
- return v0 - v1;
- }
-
public void changeIndexWithAdds(Random random, Directory dir, Version nameVersion) throws IOException {
// open writer
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random))
@@ -1023,7 +1020,7 @@ public class TestBackwardsCompatibility
assertTrue("codec used in " + name + " (" + codec.getName() + ") is not a default codec (does not begin with Lucene)",
codec.getName().startsWith("Lucene"));
}
- r.close();
+ r.close();
}
public void testAllIndexesUseDefaultCodec() throws Exception {
@@ -1186,7 +1183,7 @@ public class TestBackwardsCompatibility
// only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
- .setMergePolicy(mp).setCommitOnClose(false);
+ .setMergePolicy(mp);
IndexWriter w = new IndexWriter(ramDir, iwc);
// add few more docs:
for(int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {
@@ -1203,7 +1200,7 @@ public class TestBackwardsCompatibility
// version) to single segment index
MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
IndexWriterConfig iwc = new IndexWriterConfig(null)
- .setMergePolicy(mp).setCommitOnClose(false);
+ .setMergePolicy(mp);
IndexWriter w = new IndexWriter(dir, iwc);
w.addIndexes(ramDir);
try {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Oct 4 15:08:32 2014
@@ -424,7 +424,7 @@ public class FieldType implements Indexa
}
/**
- * Set's the field's DocValuesType
+ * Sets the field's DocValuesType
* @param type DocValues type, or null if no DocValues should be stored.
* @throws IllegalStateException if this FieldType is frozen against
* future modifications.
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Oct 4 15:08:32 2014
@@ -17,12 +17,14 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.NumberFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -35,10 +37,13 @@ import org.apache.lucene.codecs.blocktre
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -60,12 +65,14 @@ import org.apache.lucene.util.Version;
* index it can take quite a long time to run.
*
* @lucene.experimental Please make a complete backup of your
- * index before using this to fix your index!
+ * index before using this to exorcise corrupted documents from your index!
*/
-public class CheckIndex {
+public class CheckIndex implements Closeable {
private PrintStream infoStream;
private Directory dir;
+ private Lock writeLock;
+ private volatile boolean closed;
/**
* Returned from {@link #checkIndex()} detailing the health and status of the index.
@@ -111,7 +118,7 @@ public class CheckIndex {
/**
* SegmentInfos instance containing only segments that
- * had no problems (this is used with the {@link CheckIndex#fixIndex}
+ * had no problems (this is used with the {@link CheckIndex#exorciseIndex}
* method to repair the index.
*/
SegmentInfos newSegments;
@@ -342,9 +349,36 @@ public class CheckIndex {
}
/** Create a new CheckIndex on the directory. */
- public CheckIndex(Directory dir) {
+ public CheckIndex(Directory dir) throws IOException {
+ this(dir, dir.makeLock(IndexWriter.WRITE_LOCK_NAME));
+ }
+
+ /**
+ * Expert: create a directory with the specified lock.
+ * This should really not be used except for unit tests!!!!
+ * It exists only to support special tests (such as TestIndexWriterExceptions*),
+ * that would otherwise be more complicated to debug if they had to close the writer
+ * for each check.
+ */
+ public CheckIndex(Directory dir, Lock writeLock) throws IOException {
this.dir = dir;
- infoStream = null;
+ this.writeLock = writeLock;
+ this.infoStream = null;
+ if (!writeLock.obtain(IndexWriterConfig.WRITE_LOCK_TIMEOUT)) { // obtain write lock
+ throw new LockObtainFailedException("Index locked for write: " + writeLock);
+ }
+ }
+
+ private void ensureOpen() {
+ if (closed) {
+ throw new AlreadyClosedException("this instance is closed");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ IOUtils.close(writeLock);
}
private boolean crossCheckTermVectors;
@@ -415,18 +449,22 @@ public class CheckIndex {
*
* <p>As this method checks every byte in the specified
* segments, on a large index it can take quite a long
- * time to run.
- *
- * <p><b>WARNING</b>: make sure
- * you only call this when the index is not opened by any
- * writer. */
+ * time to run. */
public Status checkIndex(List<String> onlySegments) throws IOException {
+ ensureOpen();
NumberFormat nf = NumberFormat.getInstance(Locale.ROOT);
SegmentInfos sis = new SegmentInfos();
Status result = new Status();
result.dir = dir;
+ String[] files = dir.listAll();
+ String lastSegmentsFile = SegmentInfos.getLastCommitSegmentsFileName(files);
+ if (lastSegmentsFile == null) {
+ throw new IndexNotFoundException("no segments* file found in " + dir + ": files: " + Arrays.toString(files));
+ }
try {
- sis.read(dir);
+ // Do not use SegmentInfos.read(Directory) since the spooky
+ // retrying it does is not necessary here (we hold the write lock):
+ sis.read(dir, lastSegmentsFile);
} catch (Throwable t) {
if (failFast) {
IOUtils.reThrow(t);
@@ -677,7 +715,7 @@ public class CheckIndex {
}
msg(infoStream, "FAILED");
String comment;
- comment = "fixIndex() would remove reference to this segment";
+ comment = "exorciseIndex() would remove reference to this segment";
msg(infoStream, " WARNING: " + comment + "; full exception:");
if (infoStream != null)
t.printStackTrace(infoStream);
@@ -2008,12 +2046,11 @@ public class CheckIndex {
* new segments file into the index, effectively removing
* all documents in broken segments from the index.
* BE CAREFUL.
- *
- * <p><b>WARNING</b>: Make sure you only call this when the
- * index is not opened by any writer. */
- public void fixIndex(Status result) throws IOException {
+ */
+ public void exorciseIndex(Status result) throws IOException {
+ ensureOpen();
if (result.partial)
- throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
+ throw new IllegalArgumentException("can only exorcise an index that was fully checked (this status checked a subset of segments)");
result.newSegments.changed();
result.newSegments.commit(result.dir);
}
@@ -2030,31 +2067,31 @@ public class CheckIndex {
return assertsOn;
}
- /** Command-line interface to check and fix an index.
+ /** Command-line interface to check and exorcise corrupt segments from an index.
<p>
Run it like this:
<pre>
- java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-verbose] [-segment X] [-segment Y]
+ java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-verbose] [-segment X] [-segment Y]
</pre>
<ul>
- <li><code>-fix</code>: actually write a new segments_N file, removing any problematic segments
+ <li><code>-exorcise</code>: actually write a new segments_N file, removing any problematic segments. *LOSES DATA*
<li><code>-segment X</code>: only check the specified
segment(s). This can be specified multiple times,
to check more than one segment, eg <code>-segment _2
- -segment _a</code>. You can't use this with the -fix
+ -segment _a</code>. You can't use this with the -exorcise
option.
</ul>
- <p><b>WARNING</b>: <code>-fix</code> should only be used on an emergency basis as it will cause
+ <p><b>WARNING</b>: <code>-exorcise</code> should only be used on an emergency basis as it will cause
documents (perhaps many) to be permanently removed from the index. Always make
a backup copy of your index before running this! Do not run this tool on an index
that is actively being written to. You have been warned!
- <p> Run without -fix, this tool will open the index, report version information
- and report any exceptions it hits and what action it would take if -fix were
- specified. With -fix, this tool will remove any segments that have issues and
+ <p> Run without -exorcise, this tool will open the index, report version information
+ and report any exceptions it hits and what action it would take if -exorcise were
+ specified. With -exorcise, this tool will remove any segments that have issues and
write a new segments_N file. This means all documents contained in the affected
segments will be removed.
@@ -2063,8 +2100,14 @@ public class CheckIndex {
corruption, else 0.
*/
public static void main(String[] args) throws IOException, InterruptedException {
+ int exitCode = doMain(args);
+ System.exit(exitCode);
+ }
+
+ // actual main: returns exit code instead of terminating JVM (for easy testing)
+ private static int doMain(String args[]) throws IOException, InterruptedException {
- boolean doFix = false;
+ boolean doExorcise = false;
boolean doCrossCheckTermVectors = false;
boolean verbose = false;
List<String> onlySegments = new ArrayList<>();
@@ -2073,8 +2116,8 @@ public class CheckIndex {
int i = 0;
while(i < args.length) {
String arg = args[i];
- if ("-fix".equals(arg)) {
- doFix = true;
+ if ("-exorcise".equals(arg)) {
+ doExorcise = true;
} else if ("-crossCheckTermVectors".equals(arg)) {
doCrossCheckTermVectors = true;
} else if (arg.equals("-verbose")) {
@@ -2082,21 +2125,21 @@ public class CheckIndex {
} else if (arg.equals("-segment")) {
if (i == args.length-1) {
System.out.println("ERROR: missing name for -segment option");
- System.exit(1);
+ return 1;
}
i++;
onlySegments.add(args[i]);
} else if ("-dir-impl".equals(arg)) {
if (i == args.length - 1) {
System.out.println("ERROR: missing value for -dir-impl option");
- System.exit(1);
+ return 1;
}
i++;
dirImpl = args[i];
} else {
if (indexPath != null) {
System.out.println("ERROR: unexpected extra argument '" + args[i] + "'");
- System.exit(1);
+ return 1;
}
indexPath = args[i];
}
@@ -2105,32 +2148,32 @@ public class CheckIndex {
if (indexPath == null) {
System.out.println("\nERROR: index path not specified");
- System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
+ System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
"\n" +
- " -fix: actually write a new segments_N file, removing any problematic segments\n" +
+ " -exorcise: actually write a new segments_N file, removing any problematic segments\n" +
" -crossCheckTermVectors: verifies that term vectors match postings; THIS IS VERY SLOW!\n" +
- " -codec X: when fixing, codec to write the new segments_N file with\n" +
+ " -codec X: when exorcising, codec to write the new segments_N file with\n" +
" -verbose: print additional details\n" +
" -segment X: only check the specified segments. This can be specified multiple\n" +
" times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
- " You can't use this with the -fix option\n" +
+ " You can't use this with the -exorcise option\n" +
" -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
"If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
"\n" +
- "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
+ "**WARNING**: -exorcise *LOSES DATA*. This should only be used on an emergency basis as it will cause\n" +
"documents (perhaps many) to be permanently removed from the index. Always make\n" +
"a backup copy of your index before running this! Do not run this tool on an index\n" +
"that is actively being written to. You have been warned!\n" +
"\n" +
- "Run without -fix, this tool will open the index, report version information\n" +
- "and report any exceptions it hits and what action it would take if -fix were\n" +
- "specified. With -fix, this tool will remove any segments that have issues and\n" +
+ "Run without -exorcise, this tool will open the index, report version information\n" +
+ "and report any exceptions it hits and what action it would take if -exorcise were\n" +
+ "specified. With -exorcise, this tool will remove any segments that have issues and\n" +
"write a new segments_N file. This means all documents contained in the affected\n" +
"segments will be removed.\n" +
"\n" +
"This tool exits with exit code 1 if the index cannot be opened or has any\n" +
"corruption, else 0.\n");
- System.exit(1);
+ return 1;
}
if (!assertsOn())
@@ -2138,58 +2181,59 @@ public class CheckIndex {
if (onlySegments.size() == 0)
onlySegments = null;
- else if (doFix) {
- System.out.println("ERROR: cannot specify both -fix and -segment");
- System.exit(1);
+ else if (doExorcise) {
+ System.out.println("ERROR: cannot specify both -exorcise and -segment");
+ return 1;
}
System.out.println("\nOpening index @ " + indexPath + "\n");
- Directory dir = null;
+ Directory directory = null;
Path path = Paths.get(indexPath);
try {
if (dirImpl == null) {
- dir = FSDirectory.open(path);
+ directory = FSDirectory.open(path);
} else {
- dir = CommandLineUtil.newFSDirectory(dirImpl, path);
+ directory = CommandLineUtil.newFSDirectory(dirImpl, path);
}
} catch (Throwable t) {
System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");
t.printStackTrace(System.out);
- System.exit(1);
- }
-
- CheckIndex checker = new CheckIndex(dir);
- checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
- checker.setInfoStream(System.out, verbose);
-
- Status result = checker.checkIndex(onlySegments);
- if (result.missingSegments) {
- System.exit(1);
+ return 1;
}
- if (!result.clean) {
- if (!doFix) {
- System.out.println("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
+ try (Directory dir = directory;
+ CheckIndex checker = new CheckIndex(dir)) {
+ checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
+ checker.setInfoStream(System.out, verbose);
+
+ Status result = checker.checkIndex(onlySegments);
+ if (result.missingSegments) {
+ return 1;
+ }
+
+ if (!result.clean) {
+ if (!doExorcise) {
+ System.out.println("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -exorcise were specified\n");
+ } else {
+ System.out.println("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
+ System.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. YOU WILL LOSE DATA. THIS IS YOUR LAST CHANCE TO CTRL+C!");
+ for(int s=0;s<5;s++) {
+ Thread.sleep(1000);
+ System.out.println(" " + (5-s) + "...");
+ }
+ System.out.println("Writing...");
+ checker.exorciseIndex(result);
+ System.out.println("OK");
+ System.out.println("Wrote new segments file \"" + result.newSegments.getSegmentsFileName() + "\"");
+ }
+ }
+ System.out.println("");
+
+ if (result.clean == true) {
+ return 0;
} else {
- System.out.println("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
- System.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
- for(int s=0;s<5;s++) {
- Thread.sleep(1000);
- System.out.println(" " + (5-s) + "...");
- }
- System.out.println("Writing...");
- checker.fixIndex(result);
- System.out.println("OK");
- System.out.println("Wrote new segments file \"" + result.newSegments.getSegmentsFileName() + "\"");
- }
- }
- System.out.println("");
-
- final int exitCode;
- if (result.clean == true)
- exitCode = 0;
- else
- exitCode = 1;
- System.exit(exitCode);
+ return 1;
+ }
+ }
}
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Oct 4 15:08:32 2014
@@ -550,7 +550,9 @@ final class DefaultIndexingChain extends
fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
}
- norms.addValue(docState.docID, similarity.computeNorm(invertState));
+ if (invertState.length != 0) {
+ norms.addValue(docState.docID, similarity.computeNorm(invertState));
+ }
}
termsHashPerField.finish();
@@ -574,9 +576,6 @@ final class DefaultIndexingChain extends
// TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
- int lastStartOffset = 0;
- int lastPosition = 0;
-
/*
* To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream
* when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses,
@@ -602,13 +601,13 @@ final class DefaultIndexingChain extends
int posIncr = invertState.posIncrAttribute.getPositionIncrement();
invertState.position += posIncr;
- if (invertState.position < lastPosition) {
+ if (invertState.position < invertState.lastPosition) {
if (posIncr == 0) {
throw new IllegalArgumentException("first position increment must be > 0 (got 0) for field '" + field.name() + "'");
}
throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
}
- lastPosition = invertState.position;
+ invertState.lastPosition = invertState.position;
if (posIncr == 0) {
invertState.numOverlap++;
}
@@ -616,11 +615,11 @@ final class DefaultIndexingChain extends
if (checkOffsets) {
int startOffset = invertState.offset + invertState.offsetAttribute.startOffset();
int endOffset = invertState.offset + invertState.offsetAttribute.endOffset();
- if (startOffset < lastStartOffset || endOffset < startOffset) {
+ if (startOffset < invertState.lastStartOffset || endOffset < startOffset) {
throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, and offsets must not go backwards "
- + "startOffset=" + startOffset + ",endOffset=" + endOffset + ",lastStartOffset=" + lastStartOffset + " for field '" + field.name() + "'");
+ + "startOffset=" + startOffset + ",endOffset=" + endOffset + ",lastStartOffset=" + invertState.lastStartOffset + " for field '" + field.name() + "'");
}
- lastStartOffset = startOffset;
+ invertState.lastStartOffset = startOffset;
}
//System.out.println(" term=" + invertState.termAttribute);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java Sat Oct 4 15:08:32 2014
@@ -39,6 +39,9 @@ public final class FieldInvertState {
int maxTermFrequency;
int uniqueTermCount;
float boost;
+ // we must track these across field instances (multi-valued case)
+ int lastStartOffset = 0;
+ int lastPosition = 0;
AttributeSource attributeSource;
OffsetAttribute offsetAttribute;
@@ -74,6 +77,8 @@ public final class FieldInvertState {
maxTermFrequency = 0;
uniqueTermCount = 0;
boost = 1.0f;
+ lastStartOffset = 0;
+ lastPosition = 0;
}
// TODO: better name?
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java Sat Oct 4 15:08:32 2014
@@ -65,6 +65,9 @@ public class FilterLeafReader extends Le
* @param in the underlying Fields instance.
*/
public FilterFields(Fields in) {
+ if (in == null) {
+ throw new NullPointerException("incoming Fields cannot be null");
+ }
this.in = in;
}
@@ -98,6 +101,9 @@ public class FilterLeafReader extends Le
* @param in the underlying Terms instance.
*/
public FilterTerms(Terms in) {
+ if (in == null) {
+ throw new NullPointerException("incoming Terms cannot be null");
+ }
this.in = in;
}
@@ -156,7 +162,12 @@ public class FilterLeafReader extends Le
* Creates a new FilterTermsEnum
* @param in the underlying TermsEnum instance.
*/
- public FilterTermsEnum(TermsEnum in) { this.in = in; }
+ public FilterTermsEnum(TermsEnum in) {
+ if (in == null) {
+ throw new NullPointerException("incoming TermsEnum cannot be null");
+ }
+ this.in = in;
+ }
@Override
public AttributeSource attributes() {
@@ -219,6 +230,9 @@ public class FilterLeafReader extends Le
* @param in the underlying DocsEnum instance.
*/
public FilterDocsEnum(DocsEnum in) {
+ if (in == null) {
+ throw new NullPointerException("incoming DocsEnum cannot be null");
+ }
this.in = in;
}
@@ -263,6 +277,9 @@ public class FilterLeafReader extends Le
* @param in the underlying DocsAndPositionsEnum instance.
*/
public FilterDocsAndPositionsEnum(DocsAndPositionsEnum in) {
+ if (in == null) {
+ throw new NullPointerException("incoming DocsAndPositionsEnum cannot be null");
+ }
this.in = in;
}
@@ -327,6 +344,9 @@ public class FilterLeafReader extends Le
*/
public FilterLeafReader(LeafReader in) {
super();
+ if (in == null) {
+ throw new NullPointerException("incoming LeafReader cannot be null");
+ }
this.in = in;
in.registerParentReader(this);
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Oct 4 15:08:32 2014
@@ -794,7 +794,15 @@ public class IndexWriter implements Clos
// segments) pending:
changed();
} else {
- segmentInfos.read(directory);
+ String[] files = directory.listAll();
+ String lastSegmentsFile = SegmentInfos.getLastCommitSegmentsFileName(files);
+ if (lastSegmentsFile == null) {
+ throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files));
+ }
+
+ // Do not use SegmentInfos.read(Directory) since the spooky
+ // retrying it does is not necessary here (we hold the write lock):
+ segmentInfos.read(directory, lastSegmentsFile);
IndexCommit commit = config.getIndexCommit();
if (commit != null) {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java Sat Oct 4 15:08:32 2014
@@ -49,8 +49,8 @@ import org.apache.lucene.util.AttributeS
* prevents this.
*
* <p>The recommended rewrite method is {@link
- * #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}: it doesn't spend CPU
- * computing unhelpful scores, and it tries to pick the most
+ * #CONSTANT_SCORE_FILTER_REWRITE}: it doesn't spend CPU
+ * computing unhelpful scores, and is the most
* performant rewrite method given the query. If you
* need scoring (like {@link FuzzyQuery}, use
* {@link TopTermsScoringBooleanQueryRewrite} which uses
@@ -58,12 +58,12 @@ import org.apache.lucene.util.AttributeS
* and not hit this limitation.
*
* Note that org.apache.lucene.queryparser.classic.QueryParser produces
- * MultiTermQueries using {@link
- * #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} by default.
+ * MultiTermQueries using {@link #CONSTANT_SCORE_FILTER_REWRITE}
+ * by default.
*/
public abstract class MultiTermQuery extends Query {
protected final String field;
- protected RewriteMethod rewriteMethod = CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
+ protected RewriteMethod rewriteMethod = CONSTANT_SCORE_FILTER_REWRITE;
/** Abstract class that defines how the query is rewritten. */
public static abstract class RewriteMethod {
@@ -104,7 +104,7 @@ public abstract class MultiTermQuery ext
* query. Note that typically such scores are
* meaningless to the user, and require non-trivial CPU
* to compute, so it's almost always better to use {@link
- * #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} instead.
+ * #CONSTANT_SCORE_FILTER_REWRITE} instead.
*
* <p><b>NOTE</b>: This rewrite method will hit {@link
* BooleanQuery.TooManyClauses} if the number of terms
@@ -208,40 +208,6 @@ public abstract class MultiTermQuery ext
topLevel.add(q, BooleanClause.Occur.SHOULD);
}
}
-
- /** A rewrite method that tries to pick the best
- * constant-score rewrite method based on term and
- * document counts from the query. If both the number of
- * terms and documents is small enough, then {@link
- * #CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE} is used.
- * Otherwise, {@link #CONSTANT_SCORE_FILTER_REWRITE} is
- * used.
- */
- public static class ConstantScoreAutoRewrite extends org.apache.lucene.search.ConstantScoreAutoRewrite {}
-
- /** Read-only default instance of {@link
- * ConstantScoreAutoRewrite}, with {@link
- * ConstantScoreAutoRewrite#setTermCountCutoff} set to
- * {@link
- * ConstantScoreAutoRewrite#DEFAULT_TERM_COUNT_CUTOFF}
- * and {@link
- * ConstantScoreAutoRewrite#setDocCountPercent} set to
- * {@link
- * ConstantScoreAutoRewrite#DEFAULT_DOC_COUNT_PERCENT}.
- * Note that you cannot alter the configuration of this
- * instance; you'll need to create a private instance
- * instead. */
- public final static RewriteMethod CONSTANT_SCORE_AUTO_REWRITE_DEFAULT = new ConstantScoreAutoRewrite() {
- @Override
- public void setTermCountCutoff(int count) {
- throw new UnsupportedOperationException("Please create a private instance");
- }
-
- @Override
- public void setDocCountPercent(double percent) {
- throw new UnsupportedOperationException("Please create a private instance");
- }
- };
/**
* Constructs a query matching terms that cannot be represented with a single
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Sat Oct 4 15:08:32 2014
@@ -19,14 +19,14 @@ package org.apache.lucene.search;
import java.io.IOException;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.DocIdSetBuilder;
/**
* A wrapper for {@link MultiTermQuery}, that exposes its
@@ -43,7 +43,7 @@ import org.apache.lucene.util.Bits;
* this is why it is not abstract.
*/
public class MultiTermQueryWrapperFilter<Q extends MultiTermQuery> extends Filter {
-
+
protected final Q query;
/**
@@ -52,7 +52,7 @@ public class MultiTermQueryWrapperFilter
protected MultiTermQueryWrapperFilter(Q query) {
this.query = query;
}
-
+
@Override
public String toString() {
// query.toString should be ok for the filter, too, if the query boost is 1.0f
@@ -77,7 +77,7 @@ public class MultiTermQueryWrapperFilter
/** Returns the field name for this query */
public final String getField() { return query.getField(); }
-
+
/**
* Returns a DocIdSet with documents that should be permitted in search
* results.
@@ -99,24 +99,13 @@ public class MultiTermQueryWrapperFilter
final TermsEnum termsEnum = query.getTermsEnum(terms);
assert termsEnum != null;
- if (termsEnum.next() != null) {
- // fill into a FixedBitSet
- final FixedBitSet bitSet = new FixedBitSet(context.reader().maxDoc());
- DocsEnum docsEnum = null;
- do {
- // System.out.println(" iter termCount=" + termCount + " term=" +
- // enumerator.term().toBytesString());
- docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
- int docid;
- while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- bitSet.set(docid);
- }
- } while (termsEnum.next() != null);
- // System.out.println(" done termCount=" + termCount);
- return bitSet;
- } else {
- return null;
+ DocIdSetBuilder builder = new DocIdSetBuilder(context.reader().maxDoc());
+ DocsEnum docs = null;
+ while (termsEnum.next() != null) {
+ docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
+ builder.or(docs);
}
+ return builder.build();
}
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java Sat Oct 4 15:08:32 2014
@@ -72,7 +72,7 @@ import org.apache.lucene.index.Term; //
* details.
*
* <p>This query defaults to {@linkplain
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}.
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}.
* With precision steps of ≤4, this query can be run with
* one of the BooleanQuery rewrite methods without changing
* BooleanQuery's default max clause count.
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java Sat Oct 4 15:08:32 2014
@@ -29,7 +29,7 @@ import org.apache.lucene.util.ToStringUt
* is built by QueryParser for input like <code>app*</code>.
*
* <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
* rewrite method. */
public class PrefixQuery extends MultiTermQuery {
private Term prefix;
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java Sat Oct 4 15:08:32 2014
@@ -45,7 +45,7 @@ public abstract class ScoringRewrite<Q e
* query. Note that typically such scores are
* meaningless to the user, and require non-trivial CPU
* to compute, so it's almost always better to use {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} instead.
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE} instead.
*
* <p><b>NOTE</b>: This rewrite method will hit {@link
* BooleanQuery.TooManyClauses} if the number of terms
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java Sat Oct 4 15:08:32 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.util.ToStringUt
* for numerical ranges; use {@link NumericRangeQuery} instead.
*
* <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
* rewrite method.
* @since 2.9
*/
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java Sat Oct 4 15:08:32 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.util.automaton.
* a Wildcard term should not start with the wildcard <code>*</code>
*
* <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
* rewrite method.
*
* @see AutomatonQuery
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Sat Oct 4 15:08:32 2014
@@ -112,7 +112,7 @@ public class DefaultSimilarity extends T
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
}
/** Implemented as <code>sqrt(freq)</code>. */
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java Sat Oct 4 15:08:32 2014
@@ -166,13 +166,20 @@ public final class Version {
public static final Version LUCENE_4_9_0 = new Version(4, 9, 0);
/**
- * Match settings and bugs in Lucene's 4.10 release.
+ * Match settings and bugs in Lucene's 4.10.0 release.
* @deprecated (5.0) Use latest
*/
@Deprecated
public static final Version LUCENE_4_10_0 = new Version(4, 10, 0);
/**
+ * Match settings and bugs in Lucene's 4.10.1 release.
+ * @deprecated (5.0) Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_4_10_1 = new Version(4, 10, 1);
+
+ /**
* Match settings and bugs in Lucene's 5.0 release.
* @deprecated (5.0) Use latest
*/
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java Sat Oct 4 15:08:32 2014
@@ -37,7 +37,7 @@ import com.carrotsearch.randomizedtestin
@Monster("takes ~ 45 minutes")
public class Test2BBinaryDocValues extends LuceneTestCase {
- // indexes Integer.MAX_VALUE docs with a fixed binary field
+ // indexes IndexWriter.MAX_DOCS docs with a fixed binary field
public void testFixedBinary() throws Exception {
BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BFixedBinary"));
if (dir instanceof MockDirectoryWrapper) {
@@ -58,7 +58,7 @@ public class Test2BBinaryDocValues exten
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.add(dvField);
- for (int i = 0; i < Integer.MAX_VALUE; i++) {
+ for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
bytes[0] = (byte)(i >> 24);
bytes[1] = (byte)(i >> 16);
bytes[2] = (byte)(i >> 8);
@@ -96,7 +96,7 @@ public class Test2BBinaryDocValues exten
dir.close();
}
- // indexes Integer.MAX_VALUE docs with a variable binary field
+ // indexes IndexWriter.MAX_DOCS docs with a variable binary field
public void testVariableBinary() throws Exception {
BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BVariableBinary"));
if (dir instanceof MockDirectoryWrapper) {
@@ -118,7 +118,7 @@ public class Test2BBinaryDocValues exten
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.add(dvField);
- for (int i = 0; i < Integer.MAX_VALUE; i++) {
+ for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
encoder.reset(bytes);
encoder.writeVInt(i % 65535); // 1, 2, or 3 bytes
data.length = encoder.getPosition();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java Sat Oct 4 15:08:32 2014
@@ -34,7 +34,7 @@ import com.carrotsearch.randomizedtestin
@Monster("takes ~ 30 minutes")
public class Test2BNumericDocValues extends LuceneTestCase {
- // indexes Integer.MAX_VALUE docs with an increasing dv field
+ // indexes IndexWriter.MAX_DOCS docs with an increasing dv field
public void testNumerics() throws Exception {
BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BNumerics"));
if (dir instanceof MockDirectoryWrapper) {
@@ -53,7 +53,7 @@ public class Test2BNumericDocValues exte
NumericDocValuesField dvField = new NumericDocValuesField("dv", 0);
doc.add(dvField);
- for (int i = 0; i < Integer.MAX_VALUE; i++) {
+ for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
dvField.setLongValue(i);
w.addDocument(doc);
if (i % 100000 == 0) {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java Sat Oct 4 15:08:32 2014
@@ -56,7 +56,7 @@ public class Test2BSortedDocValues exten
SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
doc.add(dvField);
- for (int i = 0; i < Integer.MAX_VALUE; i++) {
+ for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
bytes[0] = (byte)(i >> 8);
bytes[1] = (byte) i;
w.addDocument(doc);
@@ -111,7 +111,7 @@ public class Test2BSortedDocValues exten
SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
doc.add(dvField);
- for (int i = 0; i < Integer.MAX_VALUE; i++) {
+ for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
bytes[0] = (byte)(i >> 24);
bytes[1] = (byte)(i >> 16);
bytes[2] = (byte)(i >> 8);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java Sat Oct 4 15:08:32 2014
@@ -42,23 +42,22 @@ import org.apache.lucene.util.AttributeI
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase.Monster;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TimeUnits;
-
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
// NOTE: SimpleText codec will consume very large amounts of
// disk (but, should run successfully). Best to run w/
// -Dtests.codec=<current codec>, and w/ plenty of RAM, eg:
//
-// ant test -Dtests.monster=true -Dtests.heapsize=8g
-//
-// java -server -Xmx8g -d64 -cp .:lib/junit-4.10.jar:./build/classes/test:./build/classes/test-framework:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=MMapDirectory -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
+// ant test -Dtests.monster=true -Dtests.heapsize=8g -Dtests.codec=Lucene50 -Dtestcase=Test2BTerms
//
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
@Monster("very slow, use 8g heap")
@TimeoutSuite(millis = 6 * TimeUnits.HOUR)
+@SuppressSysoutChecks(bugUrl="We.print.lots.o.stuff.on.purpose")
public class Test2BTerms extends LuceneTestCase {
private final static int TOKEN_LEN = 5;
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java Sat Oct 4 15:08:32 2014
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.Token;
@@ -95,6 +96,7 @@ public class TestCheckIndex extends Luce
onlySegments.add("_0");
assertTrue(checker.checkIndex(onlySegments).clean == true);
+ checker.close();
dir.close();
}
@@ -115,4 +117,22 @@ public class TestCheckIndex extends Luce
iw.close();
dir.close(); // checkindex
}
+
+ public void testObtainsLock() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+ iw.addDocument(new Document());
+ iw.commit();
+
+ // keep IW open...
+ try {
+ new CheckIndex(dir);
+ fail("should not have obtained write lock");
+ } catch (LockObtainFailedException expected) {
+ // ok
+ }
+
+ iw.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Oct 4 15:08:32 2014
@@ -33,6 +33,7 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
@@ -2783,6 +2784,7 @@ public class TestIndexWriter extends Luc
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
String s = bos.toString(IOUtils.UTF_8);
+ checker.close();
// Make sure CheckIndex didn't fail
assertTrue(s, indexStatus != null && indexStatus.clean);
@@ -2799,5 +2801,19 @@ public class TestIndexWriter extends Luc
ids.add(id);
}
}
+
+ public void testEmptyNorm() throws Exception {
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new TextField("foo", new CannedTokenStream()));
+ w.addDocument(doc);
+ w.commit();
+ w.close();
+ DirectoryReader r = DirectoryReader.open(d);
+ assertEquals(0, getOnlySegmentReader(r).getNormValues("foo").get(0));
+ r.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat Oct 4 15:08:32 2014
@@ -1194,6 +1194,7 @@ public class TestIndexWriterDelete exten
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
assertTrue(indexStatus.clean);
+ checker.close();
String s = bos.toString(IOUtils.UTF_8);
// Segment should have deletions:
@@ -1204,9 +1205,11 @@ public class TestIndexWriterDelete exten
w.close();
bos = new ByteArrayOutputStream(1024);
+ checker = new CheckIndex(dir);
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
indexStatus = checker.checkIndex(null);
assertTrue(indexStatus.clean);
+ checker.close();
s = bos.toString(IOUtils.UTF_8);
assertFalse(s.contains("has deletions"));
dir.close();
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Sat Oct 4 15:08:32 2014
@@ -426,6 +426,16 @@ public class TestPostingsOffsets extends
//expected
}
}
+
+ public void testIllegalOffsetsAcrossFieldInstances() throws Exception {
+ try {
+ checkTokens(new Token[] { makeToken("use", 1, 150, 160) },
+ new Token[] { makeToken("use", 1, 50, 60) });
+ fail();
+ } catch (IllegalArgumentException expected) {
+ //expected
+ }
+ }
public void testBackwardsOffsets() throws Exception {
try {
@@ -508,6 +518,33 @@ public class TestPostingsOffsets extends
}
// TODO: more tests with other possibilities
+ private void checkTokens(Token[] field1, Token[] field2) throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
+ boolean success = false;
+ try {
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ // store some term vectors for the checkindex cross-check
+ ft.setStoreTermVectors(true);
+ ft.setStoreTermVectorPositions(true);
+ ft.setStoreTermVectorOffsets(true);
+
+ Document doc = new Document();
+ doc.add(new Field("body", new CannedTokenStream(field1), ft));
+ doc.add(new Field("body", new CannedTokenStream(field2), ft));
+ riw.addDocument(doc);
+ riw.close();
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(dir);
+ } else {
+ IOUtils.closeWhileHandlingException(riw, dir);
+ }
+ }
+ }
+
private void checkTokens(Token[] tokens) throws IOException {
Directory dir = newDirectory();
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java Sat Oct 4 15:08:32 2014
@@ -17,6 +17,9 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.RejectedExecutionException;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@@ -65,6 +68,9 @@ public class TestReaderClosed extends Lu
searcher.search(query, 5);
} catch (AlreadyClosedException ace) {
// expected
+ } catch (RejectedExecutionException ree) {
+ // expected if the searcher has been created with threads since LuceneTestCase
+ // closes the thread-pool in a reader close listener
}
}
@@ -81,7 +87,14 @@ public class TestReaderClosed extends Lu
reader.close(); // close original child reader
try {
searcher.search(query, 5);
- } catch (AlreadyClosedException ace) {
+ } catch (Exception e) {
+ AlreadyClosedException ace = null;
+ for (Throwable t = e; t != null; t = t.getCause()) {
+ if (t instanceof AlreadyClosedException) {
+ ace = (AlreadyClosedException) t;
+ }
+ }
+ assertNotNull("Query failed, but not due to an AlreadyClosedException", ace);
assertEquals(
"this IndexReader cannot be used anymore as one of its child readers was closed",
ace.getMessage()
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java Sat Oct 4 15:08:32 2014
@@ -98,9 +98,6 @@ public class TestAutomatonQuery extends
query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
assertEquals(expected, automatonQueryNrHits(query));
-
- query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
- assertEquals(expected, automatonQueryNrHits(query));
}
/**
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java Sat Oct 4 15:08:32 2014
@@ -110,9 +110,6 @@ public class TestAutomatonQueryUnicode e
query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
assertEquals(expected, automatonQueryNrHits(query));
-
- query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
- assertEquals(expected, automatonQueryNrHits(query));
}
/**