You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/02/09 00:53:25 UTC
svn commit: r1658277 [2/38] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/lucene/replicator/
dev-tools/maven/solr/ dev-tools/maven/solr/contrib/v...
Modified: lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml (original)
+++ lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml Sun Feb 8 23:53:14 2015
@@ -25,5 +25,6 @@
<orderEntry type="module" module-name="solrj" />
<orderEntry type="module" module-name="analysis-common" />
<orderEntry type="module" module-name="lucene-core" />
+ <orderEntry type="module" scope="TEST" module-name="join" />
</component>
</module>
Modified: lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/velocity/velocity.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/velocity/velocity.iml?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/velocity/velocity.iml (original)
+++ lucene/dev/branches/lucene6005/dev-tools/idea/solr/contrib/velocity/velocity.iml Sun Feb 8 23:53:14 2015
@@ -6,8 +6,10 @@
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/velocity" type="java-test-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test-files" type="java-test-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/resources" type="java-resource" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
Modified: lucene/dev/branches/lucene6005/dev-tools/maven/lucene/replicator/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/maven/lucene/replicator/pom.xml.template?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/maven/lucene/replicator/pom.xml.template (original)
+++ lucene/dev/branches/lucene6005/dev-tools/maven/lucene/replicator/pom.xml.template Sun Feb 8 23:53:14 2015
@@ -68,11 +68,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemPropertyVariables>
- <tests.jettyConnector>${tests.jettyConnector}</tests.jettyConnector>
- </systemPropertyVariables>
- </configuration>
</plugin>
</plugins>
</build>
Modified: lucene/dev/branches/lucene6005/dev-tools/maven/solr/contrib/velocity/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/maven/solr/contrib/velocity/pom.xml.template?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/maven/solr/contrib/velocity/pom.xml.template (original)
+++ lucene/dev/branches/lucene6005/dev-tools/maven/solr/contrib/velocity/pom.xml.template Sun Feb 8 23:53:14 2015
@@ -62,12 +62,23 @@
</dependencies>
<build>
<sourceDirectory>${module-path}/src/java</sourceDirectory>
+ <resources>
+ <resource>
+ <directory>${module-path}/src/resources</directory>
+ </resource>
+ </resources>
<testSourceDirectory>${module-path}/src/test</testSourceDirectory>
<testResources>
<testResource>
<directory>${module-path}/src/test-files</directory>
</testResource>
<testResource>
+ <directory>${module-path}/src/test</directory>
+ <includes>
+ <include>velocity/*.properties</include>
+ </includes>
+ </testResource>
+ <testResource>
<directory>${top-level}/dev-tools/maven/solr</directory>
<includes>
<include>maven.testlogging.properties</include>
Modified: lucene/dev/branches/lucene6005/dev-tools/maven/solr/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/maven/solr/pom.xml.template?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/maven/solr/pom.xml.template (original)
+++ lucene/dev/branches/lucene6005/dev-tools/maven/solr/pom.xml.template Sun Feb 8 23:53:14 2015
@@ -103,7 +103,6 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
- <tests.jettyConnector>${tests.jettyConnector}</tests.jettyConnector>
<tests.disableHdfs>${tests.disableHdfs}</tests.disableHdfs>
</systemPropertyVariables>
</configuration>
Modified: lucene/dev/branches/lucene6005/dev-tools/scripts/buildAndPushRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/scripts/buildAndPushRelease.py?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/scripts/buildAndPushRelease.py (original)
+++ lucene/dev/branches/lucene6005/dev-tools/scripts/buildAndPushRelease.py Sun Feb 8 23:53:14 2015
@@ -280,9 +280,9 @@ def main():
rev = open('rev.txt', encoding='UTF-8').read()
if c.push_remote:
- url = push(version, root, rev, rcNum, username)
+ url = push(c.version, c.root, rev, c.rc_num, c.push_remote)
elif c.push_local:
- url = pushLocal(version, root, rev, c.rc_num, c.push_local)
+ url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
else:
url = None
Modified: lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavaDocs.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavaDocs.py?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavaDocs.py (original)
+++ lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavaDocs.py Sun Feb 8 23:53:14 2015
@@ -80,8 +80,6 @@ def checkClassDetails(fullPath):
Checks for invalid HTML in the full javadocs under each field/method.
"""
- isAttributeSource = fullPath.endswith('AttributeSource.html')
-
# TODO: only works with java7 generated javadocs now!
with open(fullPath, encoding='UTF-8') as f:
desc = None
@@ -90,15 +88,9 @@ def checkClassDetails(fullPath):
errors = []
for line in f.readlines():
- if isAttributeSource:
- # Work around Javadocs bug that fails to escape the <T> type parameter in {@link #getAttribute} and {@link #addAttribute}
- line = line.replace('<code>getAttribute(java.lang.Class<T>)</code>', '<code>getAttribute(java.lang.Class)</code>')
- line = line.replace('<code>addAttribute(java.lang.Class<T>)</code>', '<code>addAttribute(java.lang.Class)</code>')
-
m = reH3.search(line)
if m is not None:
if desc is not None:
- # Have to fake <ul> context because we pulled a fragment out "across" two <ul>s:
desc = ''.join(desc)
if True or cat == 'Constructor Detail':
idx = desc.find('</div>')
@@ -108,6 +100,7 @@ def checkClassDetails(fullPath):
continue
desc = desc[:idx+6]
else:
+ # Have to fake <ul> context because we pulled a fragment out "across" two <ul>s:
desc = '<ul>%s</ul>' % ''.join(desc)
#print(' VERIFY %s: %s: %s' % (cat, item, desc))
try:
@@ -123,7 +116,13 @@ def checkClassDetails(fullPath):
if m is not None:
if desc is not None:
# Have to fake <ul> context because we pulled a fragment out "across" two <ul>s:
- desc = '<ul>%s</ul>' % ''.join(desc)
+ if cat == 'Element Detail':
+ desc = ''.join(desc)
+ idx = desc.find('</dl>')
+ if idx != -1:
+ desc = desc[:idx+5]
+ else:
+ desc = '<ul>%s</ul>' % ''.join(desc)
#print(' VERIFY %s: %s: %s' % (cat, item, desc))
try:
verifyHTML(desc)
Modified: lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavadocLinks.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavadocLinks.py?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavadocLinks.py (original)
+++ lucene/dev/branches/lucene6005/dev-tools/scripts/checkJavadocLinks.py Sun Feb 8 23:53:14 2015
@@ -197,6 +197,9 @@ def checkAll(dirName):
elif link == 'http://lucene.apache.org/solr/':
# OK
pass
+ elif link == 'http://lucene.apache.org/solr/resources.html':
+ # OK
+ pass
elif link.find('lucene.apache.org/java/docs/discussion.html') != -1:
# OK
pass
@@ -206,10 +209,7 @@ def checkAll(dirName):
elif link.find('lucene.apache.org/solr/mirrors-solr-latest-redir.html') != -1:
# OK
pass
- elif link.find('lucene.apache.org/solr/discussion.html') != -1:
- # OK
- pass
- elif link.find('lucene.apache.org/solr/features.html') != -1:
+ elif link.find('lucene.apache.org/solr/quickstart.html') != -1:
# OK
pass
elif (link.find('svn.apache.org') != -1
Modified: lucene/dev/branches/lucene6005/dev-tools/scripts/smokeTestRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/dev-tools/scripts/smokeTestRelease.py?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/dev-tools/scripts/smokeTestRelease.py (original)
+++ lucene/dev/branches/lucene6005/dev-tools/scripts/smokeTestRelease.py Sun Feb 8 23:53:14 2015
@@ -635,8 +635,6 @@ def verifyUnpacked(java, project, artifa
textFiles.extend(('JRE_VERSION_MIGRATION', 'CHANGES', 'MIGRATE', 'SYSTEM_REQUIREMENTS'))
if isSrc:
textFiles.append('BUILD')
- elif not isSrc:
- textFiles.append('SYSTEM_REQUIREMENTS')
for fileName in textFiles:
fileName += '.txt'
@@ -686,8 +684,6 @@ def verifyUnpacked(java, project, artifa
if project == 'lucene':
if len(l) > 0:
raise RuntimeError('%s: unexpected files/dirs in artifact %s: %s' % (project, artifact, l))
- elif isSrc and not os.path.exists('%s/solr/SYSTEM_REQUIREMENTS.txt' % unpackPath):
- raise RuntimeError('%s: solr/SYSTEM_REQUIREMENTS.txt does not exist in artifact %s' % (project, artifact))
if isSrc:
print(' make sure no JARs/WARs in src dist...')
@@ -856,7 +852,7 @@ def testSolrExample(unpackPath, javaPath
print(' startup done')
# Create the techproducts config (used to be collection1)
- subprocess.call(['bin/solr','create_core','-n','techproducts','-c','sample_techproducts_configs'])
+ subprocess.call(['bin/solr','create_core','-c','techproducts','-d','sample_techproducts_configs'])
os.chdir('example')
print(' test utf8...')
run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log')
Modified: lucene/dev/branches/lucene6005/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/CHANGES.txt?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6005/lucene/CHANGES.txt Sun Feb 8 23:53:14 2015
@@ -29,6 +29,33 @@ API Changes
* LUCENE-6067: Accountable.getChildResources has a default
implementation returning the empty list. (Robert Muir)
+======================= Lucene 5.1.0 =======================
+
+Optimizations
+
+* LUCENE-6183, LUCENE-5647: Avoid recompressing stored fields
+ and term vectors when merging segments without deletions.
+ Lucene50Codec's BEST_COMPRESSION mode uses a higher deflate
+ level for more compact storage. (Robert Muir)
+
+* LUCENE-6184: Make BooleanScorer only score windows that contain
+ matches. (Adrien Grand)
+
+* LUCENE-6161: Speed up resolving of deleted terms to docIDs by doing
+ a combined merge sort between deleted terms and segment terms
+ instead of a separate merge sort for each segment. In delete-heavy
+ use cases this can be a sizable speedup. (Mike McCandless)
+
+API Changes
+
+* LUCENE-6204, LUCENE-6208: Simplify CompoundFormat: remove files()
+ and remove files parameter to write(). (Robert Muir)
+
+Other
+
+* LUCENE-6193: Collapse identical catch branches in try-catch statements.
+ (shalin)
+
======================= Lucene 5.0.0 =======================
New Features
@@ -130,6 +157,22 @@ New Features
* LUCENE-5914: Add an option to Lucene50Codec to support either BEST_SPEED
or BEST_COMPRESSION for stored fields. (Adrien Grand, Robert Muir)
+* LUCENE-6119: Add auto-IO-throttling to ConcurrentMergeScheduler, to
+ rate limit IO writes for each merge depending on incoming merge
+ rate. (Mike McCandless)
+
+* LUCENE-6155: Add payload support to MemoryIndex. The default highlighter's
+ QueryScorer and WeighedSpanTermExtractor now have setUsePayloads(bool).
+ (David Smiley)
+
+* LUCENE-6166: Deletions (alone) can now trigger new merges. (Mike McCandless)
+
+* LUCENE-6177: Add CustomAnalyzer that allows to configure analyzers
+ like you do in Solr's index schema. This class has a builder API to configure
+ Tokenizers, TokenFilters, and CharFilters based on their SPI names
+ and parameters as documented by the corresponding factories.
+ (Uwe Schindler)
+
Optimizations
* LUCENE-5960: Use a more efficient bitset, not a Set<Integer>, to
@@ -189,6 +232,9 @@ Optimizations
* LUCENE-6145: Make EarlyTerminatingSortingCollector able to early-terminate
when the sort order is a prefix of the index-time order. (Adrien Grand)
+* LUCENE-6178: Score boolean queries containing MUST_NOT clauses with BooleanScorer2,
+ to use skip list data and avoid unnecessary scoring. (Adrien Grand, Robert Muir)
+
API Changes
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
@@ -217,6 +263,9 @@ API Changes
* LUCENE-5527: The Collector API has been refactored to use a dedicated Collector
per leaf. (Shikhar Bhushan, Adrien Grand)
+* LUCENE-5702: The FieldComparator API has been refactor to a per-leaf API, just
+ like Collectors. (Adrien Grand)
+
* LUCENE-4246: IndexWriter.close now always closes, even if it throws
an exception. The new IndexWriterConfig.setCommitOnClose (default
true) determines whether close() should commit before closing.
@@ -332,6 +381,22 @@ API Changes
* LUCENE-6146: Replaced Directory.copy() with Directory.copyFrom().
(Robert Muir)
+* LUCENE-6149: Infix suggesters' highlighting and allTermsRequired can
+ be set at the constructor for non-contextual lookup.
+ (Boon Low, Tomás Fernández Löbbe)
+
+* LUCENE-6158, LUCENE-6165: IndexWriter.addIndexes(IndexReader...) changed to
+ addIndexes(CodecReader...) (Robert Muir)
+
+* LUCENE-6179: Out-of-order scoring is not allowed anymore, so
+ Weight.scoresDocsOutOfOrder and LeafCollector.acceptsDocsOutOfOrder have been
+ removed and boolean queries now always score in order.
+
+* LUCENE-6212: IndexWriter no longer accepts per-document Analyzer to
+ add/updateDocument. These methods were trappy as they made it
+ easy to accidentally index tokens that were not easily
+ searchable. (Mike McCandless)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
@@ -351,7 +416,7 @@ Bug Fixes
* LUCENE-5980: Don't let document length overflow. (Robert Muir)
-* LUCENE-5961: Fix the exists() method for FunctionValues returned by many ValueSoures to
+* LUCENE-5961: Fix the exists() method for FunctionValues returned by many ValueSources to
behave properly when wrapping other ValueSources which do not exist for the specified document
(hossman)
@@ -408,7 +473,26 @@ Bug Fixes
* LUCENE-6152: Fix double close problems in OutputStreamIndexOutput.
(Uwe Schindler)
+
+* LUCENE-6139: Highlighter: TokenGroup start & end offset getters should have
+ been returning the offsets of just the matching tokens in the group when
+ there's a distinction. (David Smiley)
+
+* LUCENE-6173: NumericTermAttribute and spatial/CellTokenStream do not clone
+ their BytesRef(Builder)s. Also equals/hashCode was missing. (Uwe Schindler)
+
+* LUCENE-6205: Fixed intermittent concurrency issue that could cause
+ FileNotFoundException when writing doc values updates at the same
+ time that a merge kicks off. (Mike McCandless)
+* LUCENE-6192: Fix int overflow corruption case in skip data for
+ high frequency terms in extremely large indices (Robert Muir, Mike
+ McCandless)
+
+* LUCENE-6093: Don't throw NullPointerException from
+ BlendedInfixSuggester for lookups that do not end in a prefix
+ token. (jane chang via Mike McCandless)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -463,11 +547,20 @@ Other
* LUCENE-5563: Removed sep layout: which has fallen behind on features and doesn't
perform as well as other options. (Robert Muir)
-* LUCENE-5858: Moved compatibility codecs to 'lucene-backward-codecs.jar'.
+* LUCENE-4086: Removed support for Lucene 3.x indexes. See migration guide for
+ more information. (Robert Muir)
+
+* LUCENE-5858: Moved Lucene 4 compatibility codecs to 'lucene-backward-codecs.jar'.
(Adrien Grand, Robert Muir)
* LUCENE-5915: Remove Pulsing postings format. (Robert Muir)
+======================= Lucene 4.10.4 ======================
+
+* LUCENE-6207: Fixed consumption of several terms enums on the same
+ sorted (set) doc values instance at the same time.
+ (Tom Shally, Robert Muir, Adrien Grand)
+
======================= Lucene 4.10.3 ======================
Bug fixes
@@ -475,6 +568,7 @@ Bug fixes
* LUCENE-3229: Overlapping ordered SpanNearQuery spans should not match.
(Ludovic Boutros, Paul Elschot, Greg Dearing, ehatcher)
+
======================= Lucene 4.10.2 ======================
Bug fixes
@@ -607,7 +701,7 @@ API Changes
* LUCENE-5883: You can now change the MergePolicy instance on a live IndexWriter,
without first closing and reopening the writer. This allows to e.g. run a special
merge with UpgradeIndexMergePolicy without reopening the writer. Also, MergePolicy
- no longer implements Closeable; if you need to release your custom MegePolicy's
+ no longer implements Closeable; if you need to release your custom MergePolicy's
resources, you need to implement close() and call it explicitly. (Shai Erera)
* LUCENE-5859: Deprecate Analyzer constructors taking Version. Use Analyzer.setVersion()
@@ -742,7 +836,7 @@ Bug fixes
Mike McCandless)
* LUCENE-5827: Make all Directory implementations correctly fail with
- IllegalArgumentException if slices are out of bounds. (Uwe SChindler)
+ IllegalArgumentException if slices are out of bounds. (Uwe Schindler)
* LUCENE-5897, LUCENE-5400: JFlex-based tokenizers StandardTokenizer and
UAX29URLEmailTokenizer tokenize extremely slowly over long sequences of
@@ -814,7 +908,7 @@ Changes in Backwards Compatibility Polic
API Changes
* LUCENE-5756: IndexWriter now implements Accountable and IW#ramSizeInBytes()
- has been deprecated infavor of IW#ramBytesUsed() (Simon Willnauer)
+ has been deprecated in favor of IW#ramBytesUsed() (Simon Willnauer)
* LUCENE-5725: MoreLikeThis#like now accepts multiple values per field.
The pre-existing method has been deprecated in favor of a variable arguments
@@ -1287,7 +1381,7 @@ Bug fixes
to obtain the lock. (Uwe Schindler, Robert Muir)
* LUCENE-5626: Fix bug in SimpleFSLockFactory's obtain() that sometimes throwed
- IOException (ERROR_ACESS_DENIED) on Windows if the lock file was created
+ IOException (ERROR_ACCESS_DENIED) on Windows if the lock file was created
concurrently. This error is now handled the same way like in NativeFSLockFactory
by returning false. (Uwe Schindler, Robert Muir, Dawid Weiss)
@@ -1403,8 +1497,8 @@ Bug Fixes
* LUCENE-5553: IndexReader#ReaderClosedListener is not always invoked when
IndexReader#close() is called or if refCount is 0. If an exception is
- thrown during interal close or on any of the close listerns some or all
- listerners might be missed. This can cause memory leaks if the core listeners
+ thrown during internal close or on any of the close listeners some or all
+ listeners might be missed. This can cause memory leaks if the core listeners
are used to clear caches. (Simon Willnauer)
Build
@@ -1420,7 +1514,7 @@ New Features
(Jack Conradson via Robert Muir)
* LUCENE-5337: Add Payload support to FileDictionary (Suggest) and make it more
- configurable (Areek Zilluer via Erick Erickson)
+ configurable (Areek Zillur via Erick Erickson)
* LUCENE-5329: suggest: DocumentDictionary and
DocumentExpressionDictionary are now lenient for dirty documents
@@ -1589,8 +1683,8 @@ Bug fixes
to byte, before calling Similarity.decodeNormValue. (Peng Cheng via
Mike McCandless)
-* LUCENE-5436: RefrenceManager#accquire can result in infinite loop if
- managed resource is abused outside of the RefrenceManager. Decrementing
+* LUCENE-5436: ReferenceManager#accquire can result in infinite loop if
+ managed resource is abused outside of the ReferenceManager. Decrementing
the reference without a corresponding incRef() call can cause an infinite
loop. ReferenceManager now throws IllegalStateException if currently managed
resources ref count is 0. (Simon Willnauer)
@@ -1598,7 +1692,7 @@ Bug fixes
* LUCENE-5443: Lucene45DocValuesProducer.ramBytesUsed() may throw
ConcurrentModificationException. (Shai Erera, Simon Willnauer)
-* LUCENE-5444: MemoryIndex did't respect the analyzers offset gap and
+* LUCENE-5444: MemoryIndex didn't respect the analyzers offset gap and
offsets were corrupted if multiple fields with the same name were
added to the memory index. (Britta Weber, Simon Willnauer)
@@ -1622,7 +1716,7 @@ API Changes
to IndexWriter.addDocument(). (Shai Erera, Gilad Barkai, Rob
Muir, Mike McCandless)
-* LUCENE-5405: Make ShingleAnalzyerWrapper.getWrappedAnalyzer() public final (gsingers)
+* LUCENE-5405: Make ShingleAnalyzerWrapper.getWrappedAnalyzer() public final (gsingers)
* LUCENE-5395: The SpatialArgsParser now only reads WKT, no more "lat, lon"
etc. but it's easy to override the parseShape method if you wish. (David
@@ -1666,7 +1760,7 @@ Documentation
to the analysis package overview.
(Benson Margulies via Robert Muir - pull request #12)
-* LUCENE-5389: Add more guidance in the analyis documentation
+* LUCENE-5389: Add more guidance in the analysis documentation
package overview.
(Benson Margulies via Robert Muir - pull request #14)
@@ -1865,7 +1959,7 @@ Optimizations
Documentation
* LUCENE-5211: Better javadocs and error checking of 'format' option in
- StopFilterFactory, as well as comments in all snowball formated files
+ StopFilterFactory, as well as comments in all snowball formatted files
about specifying format option. (hossman)
Changes in backwards compatibility policy
@@ -2023,7 +2117,7 @@ Bug Fixes
some result documents were missing category associations. (Shai Erera)
* LUCENE-5152: Fix MemoryPostingsFormat to not modify borrowed BytesRef from FSTEnum
- seek/lookup which can cause sideeffects if done on a cached FST root arc.
+ seek/lookup which can cause side effects if done on a cached FST root arc.
(Simon Willnauer)
* LUCENE-5160: Handle the case where reading from a file or FileChannel returns -1,
@@ -2096,7 +2190,7 @@ API Changes
* LUCENE-5144: StandardFacetsAccumulator renamed to OldFacetsAccumulator, and all
associated classes were moved under o.a.l.facet.old. The intention to remove it
- one day, when the features it covers (complements, partitiona, sampling) will be
+ one day, when the features it covers (complements, partitions, sampling) will be
migrated to the new FacetsAggregator and FacetsAccumulator API. Also,
FacetRequest.createAggregator was replaced by OldFacetsAccumulator.createAggregator.
(Shai Erera)
@@ -2173,7 +2267,7 @@ Optimizations
synchronization and concurrent interaction with IndexWriter. DWPT is now
only setup once and has no reset logic. All segment publishing and state
transition from DWPT into IndexWriter is now done via an Event-Queue
- processed from within the IndexWriter in order to prevent suituations
+ processed from within the IndexWriter in order to prevent situations
where DWPT or DW calling int IW causing deadlocks. (Simon Willnauer)
* LUCENE-5182: Terminate phrase searches early if max phrase window is
@@ -2294,7 +2388,7 @@ Changes in backwards compatibility polic
jvms and it's faster not to cache. (Robert Muir)
* LUCENE-5038: MergePolicy now has a default implementation for useCompoundFile based
- on segment size and noCFSRatio. The default implemantion was pulled up from
+ on segment size and noCFSRatio. The default implementation was pulled up from
TieredMergePolicy. (Simon Willnauer)
* LUCENE-5063: FieldCache.get(Bytes|Shorts), SortField.Type.(BYTE|SHORT) and
@@ -2494,7 +2588,7 @@ API Changes
* LUCENE-5097: Analyzer now has an additional tokenStream(String fieldName,
String text) method, so wrapping by StringReader for common use is no
- longer needed. This method uses an internal reuseable reader, which was
+ longer needed. This method uses an internal reusable reader, which was
previously only used by the Field class. (Uwe Schindler, Robert Muir)
* LUCENE-4542: HunspellStemFilter's maximum recursion level is now configurable.
@@ -2503,7 +2597,7 @@ API Changes
Build
* LUCENE-4987: Upgrade randomized testing to version 2.0.10:
- Test framework may fail internally due to overly aggresive J9 optimizations.
+ Test framework may fail internally due to overly aggressive J9 optimizations.
(Dawid Weiss, Shai Erera)
* LUCENE-5043: The eclipse target now uses the containing directory for the
@@ -2550,7 +2644,7 @@ Bug Fixes
large heap/explicitly disabled. (Mike McCandless, Uwe Schindler, Robert Muir)
* LUCENE-4953: Fixed ParallelCompositeReader to inform ReaderClosedListeners of
- its synthetic subreaders. FieldCaches keyed on the atomic childs will be purged
+ its synthetic subreaders. FieldCaches keyed on the atomic children will be purged
earlier and FC insanity prevented. In addition, ParallelCompositeReader's
toString() was changed to better reflect the reader structure.
(Mike McCandless, Uwe Schindler)
@@ -2915,7 +3009,7 @@ Bug Fixes
OpenMode.CREATE_OR_APPEND is used. This might also affect application that set
the open mode manually using DirectoryReader#indexExists. (Simon Willnauer)
-* LUCENE-4878: Override getRegexpQuery in MultiFieldQueryParser to prefent
+* LUCENE-4878: Override getRegexpQuery in MultiFieldQueryParser to prevent
NullPointerException when regular expression syntax is used with
MultiFieldQueryParser. (Simon Willnauer, Adam Rauch)
@@ -2984,7 +3078,7 @@ Changes in backwards compatibility polic
sorted by value, while ties are broken by category ordinal. (Shai Erera)
* LUCENE-4772: Facet associations moved to new FacetsAggregator API. You
- should override FacetsAccumualtor and return the relevant aggregator,
+ should override FacetsAccumulator and return the relevant aggregator,
for aggregating the association values. (Shai Erera)
* LUCENE-4748: A FacetRequest on a non-existent field now returns an
@@ -3675,7 +3769,7 @@ API Changes
* LUCENE-4391, LUCENE-4440: All methods of Lucene40Codec but
getPostingsFormatForField are now final. To reuse functionality
of Lucene40, you should extend FilterCodec and delegate to Lucene40
- instead of extending Lucene40Codec. (Adrien Grand, Shai Erea,
+ instead of extending Lucene40Codec. (Adrien Grand, Shai Erera,
Robert Muir, Uwe Schindler)
* LUCENE-4299: Added Terms.hasPositions() and Terms.hasOffsets().
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/README.txt?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/README.txt (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/README.txt Sun Feb 8 23:53:14 2015
@@ -1,7 +1,7 @@
-Lucene Contrib Analyzers README file
+Lucene Analyzers README file
This project provides pre-compiled version of the Snowball stemmers
-based on revision 500 of the Tartarus Snowball repository,
+based on revision 502 of the Tartarus Snowball repository,
together with classes integrating them with the Lucene search engine.
A few changes has been made to the static Snowball code and compiled stemmers:
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -48,7 +48,6 @@ public class DictionaryCompoundWordToken
/** Creates a new DictionaryCompoundWordTokenFilterFactory */
public DictionaryCompoundWordTokenFilterFactory(Map<String, String> args) {
super(args);
- assureMatchVersion();
dictFile = require(args, "dictionary");
minWordSize = getInt(args, "minWordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE);
minSubwordSize = getInt(args, "minSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -74,7 +74,6 @@ public class HyphenationCompoundWordToke
/** Creates a new HyphenationCompoundWordTokenFilterFactory */
public HyphenationCompoundWordTokenFilterFactory(Map<String, String> args) {
super(args);
- assureMatchVersion();
dictFile = get(args, "dictionary");
encoding = get(args, "encoding");
hypFile = require(args, "hyphenator");
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -78,7 +78,6 @@ public class CapitalizationFilterFactory
/** Creates a new CapitalizationFilterFactory */
public CapitalizationFilterFactory(Map<String, String> args) {
super(args);
- assureMatchVersion();
boolean ignoreCase = getBoolean(args, KEEP_IGNORE_CASE, false);
Set<String> k = getSet(args, KEEP);
if (k != null) {
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -55,7 +55,6 @@ public class StemmerOverrideFilterFactor
@Override
public void inform(ResourceLoader loader) throws IOException {
if (dictionaryFiles != null) {
- assureMatchVersion();
List<String> files = splitFileNames(dictionaryFiles);
if (files.size() > 0) {
StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(ignoreCase);
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -63,7 +63,6 @@ public class WordDelimiterFilterFactory
/** Creates a new WordDelimiterFilterFactory */
public WordDelimiterFilterFactory(Map<String, String> args) {
super(args);
- assureMatchVersion();
int flags = 0;
if (getInt(args, "generateWordParts", 1) != 0) {
flags |= GENERATE_WORD_PARTS;
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java Sun Feb 8 23:53:14 2015
@@ -102,7 +102,6 @@ public class SynonymFilterFactory extend
}
if (tokenizerFactory != null) {
- assureMatchVersion();
tokArgs.put("luceneMatchVersion", getLuceneMatchVersion().toString());
for (Iterator<String> itr = args.keySet().iterator(); itr.hasNext();) {
String key = itr.next();
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java Sun Feb 8 23:53:14 2015
@@ -70,7 +70,7 @@ public abstract class AbstractAnalysisFa
originalArgs = Collections.unmodifiableMap(new HashMap<>(args));
String version = get(args, LUCENE_MATCH_VERSION_PARAM);
if (version == null) {
- luceneMatchVersion = null;
+ luceneMatchVersion = Version.LATEST;
} else {
try {
luceneMatchVersion = Version.parseLeniently(version);
@@ -85,16 +85,6 @@ public abstract class AbstractAnalysisFa
return originalArgs;
}
- /** this method can be called in the {@link org.apache.lucene.analysis.util.TokenizerFactory#create()}
- * or {@link org.apache.lucene.analysis.util.TokenFilterFactory#create(org.apache.lucene.analysis.TokenStream)} methods,
- * to inform user, that for this factory a {@link #luceneMatchVersion} is required */
- protected final void assureMatchVersion() {
- if (luceneMatchVersion == null) {
- throw new IllegalArgumentException("Configuration Error: Factory '" + this.getClass().getName() +
- "' needs a 'luceneMatchVersion' parameter");
- }
- }
-
public final Version getLuceneMatchVersion() {
return this.luceneMatchVersion;
}
@@ -241,7 +231,6 @@ public abstract class AbstractAnalysisFa
*/
protected final CharArraySet getWordSet(ResourceLoader loader,
String wordFiles, boolean ignoreCase) throws IOException {
- assureMatchVersion();
List<String> files = splitFileNames(wordFiles);
CharArraySet words = null;
if (files.size() > 0) {
@@ -267,7 +256,6 @@ public abstract class AbstractAnalysisFa
* except the input is in snowball format. */
protected final CharArraySet getSnowballWordSet(ResourceLoader loader,
String wordFiles, boolean ignoreCase) throws IOException {
- assureMatchVersion();
List<String> files = splitFileNames(wordFiles);
CharArraySet words = null;
if (files.size() > 0) {
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/overview.html?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/overview.html (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/java/overview.html Sun Feb 8 23:53:14 2015
@@ -27,7 +27,8 @@
This module contains concrete components ({@link org.apache.lucene.analysis.CharFilter}s,
{@link org.apache.lucene.analysis.Tokenizer}s, and ({@link org.apache.lucene.analysis.TokenFilter}s) for
analyzing different types of content. It also provides a number of {@link org.apache.lucene.analysis.Analyzer}s
- for different languages that you can use to get started quickly.
+ for different languages that you can use to get started quickly. To define fully custom Analyzers
+ (like in the index schema of Apache Solr), this module provides {@link org.apache.lucene.analysis.custom.CustomAnalyzer}.
</p>
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java Sun Feb 8 23:53:14 2015
@@ -135,9 +135,7 @@ public class TestFactories extends BaseT
AbstractAnalysisFactory factory = null;
try {
factory = ctor.newInstance(args);
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
+ } catch (InstantiationException | IllegalAccessException e) {
throw new RuntimeException(e);
} catch (InvocationTargetException e) {
if (e.getCause() instanceof IllegalArgumentException) {
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Sun Feb 8 23:53:14 2015
@@ -712,10 +712,8 @@ public class TestRandomChains extends Ba
} else {
Rethrow.rethrow(cause);
}
- } catch (IllegalAccessException iae) {
+ } catch (IllegalAccessException | InstantiationException iae) {
Rethrow.rethrow(iae);
- } catch (InstantiationException ie) {
- Rethrow.rethrow(ie);
}
return null; // no success
}
Modified: lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java Sun Feb 8 23:53:14 2015
@@ -17,11 +17,8 @@ package org.apache.lucene.analysis.util;
* limitations under the License.
*/
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -29,8 +26,6 @@ import java.nio.file.Path;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
public class TestFilesystemResourceLoader extends LuceneTestCase {
Modified: lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java Sun Feb 8 23:53:14 2015
@@ -58,9 +58,7 @@ public final class UIMAAnnotationsTokeni
protected void initializeIterator() throws IOException {
try {
analyzeInput();
- } catch (AnalysisEngineProcessException e) {
- throw new IOException(e);
- } catch (ResourceInitializationException e) {
+ } catch (AnalysisEngineProcessException | ResourceInitializationException e) {
throw new IOException(e);
}
finalOffset = correctOffset(cas.getDocumentText().length());
Modified: lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java Sun Feb 8 23:53:14 2015
@@ -70,9 +70,7 @@ public final class UIMATypeAwareAnnotati
protected void initializeIterator() throws IOException {
try {
analyzeInput();
- } catch (AnalysisEngineProcessException e) {
- throw new IOException(e);
- } catch (ResourceInitializationException e) {
+ } catch (AnalysisEngineProcessException | ResourceInitializationException e) {
throw new IOException(e);
}
featurePath = cas.createFeaturePath();
Modified: lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene6005/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Feb 8 23:53:14 2015
@@ -297,6 +297,8 @@ public class TestBackwardsCompatibility
"4.10.1-nocfs",
"4.10.2-cfs",
"4.10.2-nocfs",
+ "4.10.3-cfs",
+ "4.10.3-nocfs",
};
final static String[] oldSingleSegmentNames = {
@@ -548,11 +550,11 @@ public class TestBackwardsCompatibility
public void testAddOldIndexesReader() throws IOException {
for (String name : oldNames) {
- IndexReader reader = DirectoryReader.open(oldIndexDirs.get(name));
+ DirectoryReader reader = DirectoryReader.open(oldIndexDirs.get(name));
Directory targetDir = newDirectory();
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
- w.addIndexes(reader);
+ TestUtil.addIndexesSlowly(w, reader);
w.close();
reader.close();
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java Sun Feb 8 23:53:14 2015
@@ -202,10 +202,8 @@ public class EnwikiContentSource extends
}
}
}
- } catch (SAXException sae) {
+ } catch (SAXException | IOException sae) {
throw new RuntimeException(sae);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
} finally {
synchronized(this) {
threadDone = true;
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/NoMoreDataException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/NoMoreDataException.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/NoMoreDataException.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/NoMoreDataException.java Sun Feb 8 23:53:14 2015
@@ -19,7 +19,7 @@ package org.apache.lucene.benchmark.byTa
/**
* Exception indicating there is no more data.
- * Thrown by Docs Makers if doc.maker.forever is false and docs sources of that maker where exhausted.
+ * Thrown by Docs Makers if content.source.forever is false and docs sources of that maker where exhausted.
* This is useful for iterating all document of a source, in case we don't know in advance how many docs there are.
*/
public class NoMoreDataException extends Exception {
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html Sun Feb 8 23:53:14 2015
@@ -557,7 +557,7 @@ Here is a list of currently defined prop
<li><b>Docs and queries creation:</b></li>
<ul><li>analyzer
</li><li>doc.maker
- </li><li>doc.maker.forever
+ </li><li>content.source.forever
</li><li>html.parser
</li><li>doc.stored
</li><li>doc.tokenized
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTask.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTask.java Sun Feb 8 23:53:14 2015
@@ -18,18 +18,23 @@ package org.apache.lucene.benchmark.byTa
*/
import java.nio.file.Paths;
+import java.util.List;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
/**
* Adds an input index to an existing index, using
* {@link IndexWriter#addIndexes(Directory...)} or
- * {@link IndexWriter#addIndexes(IndexReader...)}. The location of the input
+ * {@link IndexWriter#addIndexes(CodecReader...)}. The location of the input
* index is specified by the parameter {@link #ADDINDEXES_INPUT_DIR} and is
* assumed to be a directory on the file system.
* <p>
@@ -63,11 +68,13 @@ public class AddIndexesTask extends Perf
if (useAddIndexesDir) {
writer.addIndexes(inputDir);
} else {
- IndexReader r = DirectoryReader.open(inputDir);
- try {
- writer.addIndexes(r);
- } finally {
- r.close();
+ try (IndexReader r = DirectoryReader.open(inputDir)) {
+ CodecReader leaves[] = new CodecReader[r.leaves().size()];
+ int i = 0;
+ for (LeafReaderContext leaf : r.leaves()) {
+ leaves[i++] = SlowCodecReaderWrapper.wrap(leaf.reader());
+ }
+ writer.addIndexes(leaves);
}
}
return 1;
@@ -79,7 +86,7 @@ public class AddIndexesTask extends Perf
* @param params
* {@code useAddIndexesDir=true} for using
* {@link IndexWriter#addIndexes(Directory...)} or {@code false} for
- * using {@link IndexWriter#addIndexes(IndexReader...)}. Defaults to
+ * using {@link IndexWriter#addIndexes(CodecReader...)}. Defaults to
* {@code true}.
*/
@Override
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java Sun Feb 8 23:53:14 2015
@@ -120,8 +120,7 @@ public abstract class ReadTask extends P
// pulling the Weight ourselves:
TopFieldCollector collector = TopFieldCollector.create(sort, numHits,
true, withScore(),
- withMaxScore(),
- false);
+ withMaxScore());
searcher.search(q, null, collector);
hits = collector.topDocs();
} else {
@@ -187,7 +186,7 @@ public abstract class ReadTask extends P
}
protected Collector createCollector() throws Exception {
- return TopScoreDocCollector.create(numHits(), true);
+ return TopScoreDocCollector.create(numHits());
}
Modified: lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java (original)
+++ lucene/dev/branches/lucene6005/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java Sun Feb 8 23:53:14 2015
@@ -52,10 +52,8 @@ public class SearchWithCollectorTask ext
@Override
protected Collector createCollector() throws Exception {
Collector collector = null;
- if (clnName.equalsIgnoreCase("topScoreDocOrdered") == true) {
- collector = TopScoreDocCollector.create(numHits(), true);
- } else if (clnName.equalsIgnoreCase("topScoreDocUnOrdered") == true) {
- collector = TopScoreDocCollector.create(numHits(), false);
+ if (clnName.equalsIgnoreCase("topScoreDoc") == true) {
+ collector = TopScoreDocCollector.create(numHits());
} else if (clnName.length() > 0){
collector = Class.forName(clnName).asSubclass(Collector.class).newInstance();
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java Sun Feb 8 23:53:14 2015
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java Sun Feb 8 23:53:14 2015
@@ -24,6 +24,7 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsReader;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
@@ -56,9 +57,8 @@ public class BlockTreeOrdsPostingsFormat
public BlockTreeOrdsPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
super("BlockTreeOrds");
this.minTermBlockSize = minTermBlockSize;
- assert minTermBlockSize > 1;
this.maxTermBlockSize = maxTermBlockSize;
- assert minTermBlockSize <= maxTermBlockSize;
+ BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java Sun Feb 8 23:53:14 2015
@@ -418,7 +418,7 @@ public final class OrdsBlockTreeTermsWri
// System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
//}
Output output = indexEnt.output;
- long blockTermCount = output.endOrd - output.startOrd + 1;
+ //long blockTermCount = output.endOrd - output.startOrd + 1;
Output newOutput = FST_OUTPUTS.newOutput(output.bytes, termOrdOffset+output.startOrd, output.endOrd-termOrdOffset);
//System.out.println(" append sub=" + indexEnt.input + " output=" + indexEnt.output + " termOrdOffset=" + termOrdOffset + " blockTermCount=" + blockTermCount + " newOutput=" + newOutput + " endOrd=" + (termOrdOffset+Long.MAX_VALUE-output.endOrd));
builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), newOutput);
@@ -603,9 +603,6 @@ public final class OrdsBlockTreeTermsWri
// compact format in this case:
boolean isLeafBlock = hasSubBlocks == false;
- // Number of terms in this block
- int termCount;
-
// Number of terms in this block and all sub-blocks (recursively)
long totalTermCount;
@@ -652,12 +649,10 @@ public final class OrdsBlockTreeTermsWri
bytesWriter.reset();
absolute = false;
}
- termCount = end-start;
totalTermCount = end-start;
} else {
// Mixed terms and sub-blocks:
subIndices = new ArrayList<>();
- termCount = 0;
totalTermCount = 0;
for (int i=start;i<end;i++) {
PendingEntry ent = pending.get(i);
@@ -705,7 +700,6 @@ public final class OrdsBlockTreeTermsWri
bytesWriter.reset();
absolute = false;
- termCount++;
totalTermCount++;
} else {
PendingBlock block = (PendingBlock) ent;
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Sun Feb 8 23:53:14 2015
@@ -305,6 +305,7 @@ class MemoryDocValuesConsumer extends Do
int maxLength = Integer.MIN_VALUE;
final long startFP = data.getFilePointer();
boolean missing = false;
+ int upto = 0;
for(BytesRef v : values) {
final int length;
if (v == null) {
@@ -314,8 +315,9 @@ class MemoryDocValuesConsumer extends Do
length = v.length;
}
if (length > MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH) {
- throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH);
+ throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH + " but got length=" + length + " v=" + v + "; upto=" + upto + " values=" + values);
}
+ upto++;
minLength = Math.min(minLength, length);
maxLength = Math.max(maxLength, length);
if (v != null) {
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java Sun Feb 8 23:53:14 2015
@@ -29,7 +29,6 @@ import java.util.Locale;
import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.MergeState.CheckAbort;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
@@ -157,11 +156,11 @@ public class SimpleTextCompoundFormat ex
}
@Override
- public void write(Directory dir, SegmentInfo si, Collection<String> files, CheckAbort checkAbort, IOContext context) throws IOException {
+ public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
- int numFiles = files.size();
- String names[] = files.toArray(new String[numFiles]);
+ int numFiles = si.files().size();
+ String names[] = si.files().toArray(new String[numFiles]);
Arrays.sort(names);
long startOffsets[] = new long[numFiles];
long endOffsets[] = new long[numFiles];
@@ -181,8 +180,6 @@ public class SimpleTextCompoundFormat ex
out.copyBytes(in, in.length());
}
endOffsets[i] = out.getFilePointer();
-
- checkAbort.work(endOffsets[i] - startOffsets[i]);
}
long tocPos = out.getFilePointer();
@@ -212,11 +209,6 @@ public class SimpleTextCompoundFormat ex
SimpleTextUtil.writeNewline(out);
}
}
-
- @Override
- public String[] files(SegmentInfo si) {
- return new String[] { IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION) };
- }
// helper method to strip strip away 'prefix' from 'scratch' and return as String
private String stripPrefix(BytesRefBuilder scratch, BytesRef prefix) throws IOException {
Modified: lucene/dev/branches/lucene6005/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/common-build.xml?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/common-build.xml (original)
+++ lucene/dev/branches/lucene6005/lucene/common-build.xml Sun Feb 8 23:53:14 2015
@@ -1042,7 +1042,6 @@
<propertyref prefix="tests.badapples" />
<propertyref prefix="tests.bwcdir" />
<propertyref prefix="tests.timeoutSuite" />
- <propertyref prefix="tests.jettyConnector" />
<propertyref prefix="tests.disableHdfs" />
<propertyref prefix="tests.filter" />
<propertyref prefix="tests.leavetmpdir" />
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/package.html?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/package.html Sun Feb 8 23:53:14 2015
@@ -164,7 +164,11 @@ and proximity searches (though sentence
supplies a large family of <code>Analyzer</code> classes that deliver useful
analysis chains. The most common of these is the <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/standard/StandardAnalyzer.html">StandardAnalyzer</a>.
Many applications will have a long and industrious life with nothing more
- than the <code>StandardAnalyzer</code>.
+ than the <code>StandardAnalyzer</code>. The <a href="{@docRoot}/../analyzers-common/overview-summary.html">analyzers-common</a>
+ library provides many pre-existing analyzers for various languages.
+ The analysis-common library also allows to configure a custom Analyzer without subclassing using the
+ <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/custom/CustomAnalyzer.html">CustomAnalyzer</a>
+ class.
</p>
<p>
Aside from the <code>StandardAnalyzer</code>,
@@ -258,8 +262,7 @@ and proximity searches (though sentence
create, or a combination of existing and newly created components. Before
pursuing this approach, you may find it worthwhile to explore the
<a href="{@docRoot}/../analyzers-common/overview-summary.html">analyzers-common</a> library and/or ask on the
- <a href="http://lucene.apache.org/core/discussion.html"
- >java-user@lucene.apache.org mailing list</a> first to see if what you
+ <a href="http://lucene.apache.org/core/discussion.html">java-user@lucene.apache.org mailing list</a> first to see if what you
need already exists. If you are still committed to creating your own
Analyzer, have a look at the source code of any one of the many samples
located in this package.
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java Sun Feb 8 23:53:14 2015
@@ -413,7 +413,7 @@ public final class CodecUtil {
final int magic = in.readInt();
if (magic != FOOTER_MAGIC) {
- throw new CorruptIndexException("codec footer mismatch: actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
+ throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
}
final int algorithmID = in.readInt();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java Sun Feb 8 23:53:14 2015
@@ -18,9 +18,7 @@ package org.apache.lucene.codecs;
*/
import java.io.IOException;
-import java.util.Collection;
-import org.apache.lucene.index.MergeState.CheckAbort;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -45,15 +43,7 @@ public abstract class CompoundFormat {
public abstract Directory getCompoundReader(Directory dir, SegmentInfo si, IOContext context) throws IOException;
/**
- * Packs the provided files into a compound format.
+ * Packs the provided segment's files into a compound format.
*/
- // TODO: get checkAbort out of here, and everywhere, and have iw do it at a higher level
- public abstract void write(Directory dir, SegmentInfo si, Collection<String> files, CheckAbort checkAbort, IOContext context) throws IOException;
-
- /**
- * Returns the compound file names used by this segment.
- */
- // TODO: get this out of here, and use trackingdirwrapper. but this is really scary in IW right now...
- // NOTE: generally si.useCompoundFile is not even yet 'set' when this is called.
- public abstract String[] files(SegmentInfo si);
+ public abstract void write(Directory dir, SegmentInfo si, IOContext context) throws IOException;
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Sun Feb 8 23:53:14 2015
@@ -97,7 +97,6 @@ public abstract class StoredFieldsWriter
storedFieldsReader.visitDocument(docID, visitor);
finishDocument();
docCount++;
- mergeState.checkAbort.work(300);
}
}
finish(mergeState.mergeFieldInfos, docCount);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Sun Feb 8 23:53:14 2015
@@ -196,7 +196,6 @@ public abstract class TermVectorsWriter
}
addAllDocVectors(vectors, mergeState);
docCount++;
- mergeState.checkAbort.work(300);
}
}
finish(mergeState.mergeFieldInfos, docCount);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Sun Feb 8 23:53:14 2015
@@ -36,6 +36,7 @@ import static org.apache.lucene.codecs.c
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.TYPE_BITS;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.TYPE_MASK;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_CURRENT;
+import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_CHUNK_STATS;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_START;
import java.io.EOFException;
@@ -88,6 +89,8 @@ public final class CompressingStoredFiel
private final int numDocs;
private final boolean merging;
private final BlockState state;
+ private final long numChunks; // number of compressed blocks written
+ private final long numDirtyChunks; // number of incomplete compressed blocks written
private boolean closed;
// used by clone
@@ -102,6 +105,8 @@ public final class CompressingStoredFiel
this.compressionMode = reader.compressionMode;
this.decompressor = reader.decompressor.clone();
this.numDocs = reader.numDocs;
+ this.numChunks = reader.numChunks;
+ this.numDirtyChunks = reader.numDirtyChunks;
this.merging = merging;
this.state = new BlockState();
this.closed = false;
@@ -145,9 +150,6 @@ public final class CompressingStoredFiel
try {
// Open the data file and read metadata
fieldsStream = d.openInput(fieldsStreamFN, context);
- if (maxPointer + CodecUtil.footerLength() != fieldsStream.length()) {
- throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.length(), fieldsStream);
- }
final String codecNameDat = formatName + CODEC_SFX_DAT;
final int fieldsVersion = CodecUtil.checkIndexHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
if (version != fieldsVersion) {
@@ -161,6 +163,17 @@ public final class CompressingStoredFiel
this.merging = false;
this.state = new BlockState();
+ if (version >= VERSION_CHUNK_STATS) {
+ fieldsStream.seek(maxPointer);
+ numChunks = fieldsStream.readVLong();
+ numDirtyChunks = fieldsStream.readVLong();
+ if (numDirtyChunks > numChunks) {
+ throw new CorruptIndexException("invalid chunk counts: dirty=" + numDirtyChunks + ", total=" + numChunks, fieldsStream);
+ }
+ } else {
+ numChunks = numDirtyChunks = -1;
+ }
+
// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
// for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
@@ -496,8 +509,6 @@ public final class CompressingStoredFiel
final int totalLength = offsets[chunkDocs];
final int numStoredFields = this.numStoredFields[index];
- fieldsStream.seek(startPointer);
-
final DataInput documentInput;
if (length == 0) {
// empty
@@ -506,6 +517,7 @@ public final class CompressingStoredFiel
// already decompressed
documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset + offset, length);
} else if (sliced) {
+ fieldsStream.seek(startPointer);
decompressor.decompress(fieldsStream, chunkSize, offset, Math.min(length, chunkSize - offset), bytes);
documentInput = new DataInput() {
@@ -545,6 +557,7 @@ public final class CompressingStoredFiel
};
} else {
+ fieldsStream.seek(startPointer);
decompressor.decompress(fieldsStream, totalLength, offset, length, bytes);
assert bytes.length == length;
documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset, bytes.length);
@@ -610,10 +623,34 @@ public final class CompressingStoredFiel
CompressionMode getCompressionMode() {
return compressionMode;
}
+
+ CompressingStoredFieldsIndexReader getIndexReader() {
+ return indexReader;
+ }
+
+ long getMaxPointer() {
+ return maxPointer;
+ }
+
+ IndexInput getFieldsStream() {
+ return fieldsStream;
+ }
int getChunkSize() {
return chunkSize;
}
+
+ long getNumChunks() {
+ return numChunks;
+ }
+
+ long getNumDirtyChunks() {
+ return numDirtyChunks;
+ }
+
+ int getPackedIntsVersion() {
+ return packedIntsVersion;
+ }
@Override
public long ramBytesUsed() {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1658277&r1=1658276&r2=1658277&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Sun Feb 8 23:53:14 2015
@@ -24,6 +24,7 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.SerializedDocument;
+import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@@ -33,6 +34,7 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
@@ -68,13 +70,15 @@ public final class CompressingStoredFiel
static final String CODEC_SFX_IDX = "Index";
static final String CODEC_SFX_DAT = "Data";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHUNK_STATS = 1;
+ static final int VERSION_CURRENT = VERSION_CHUNK_STATS;
private final String segment;
private CompressingStoredFieldsIndexWriter indexWriter;
private IndexOutput fieldsStream;
private final Compressor compressor;
+ private final CompressionMode compressionMode;
private final int chunkSize;
private final int maxDocsPerChunk;
@@ -83,12 +87,16 @@ public final class CompressingStoredFiel
private int[] endOffsets; // end offsets in bufferedDocs
private int docBase; // doc ID at the beginning of the chunk
private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
+
+ private long numChunks; // number of compressed blocks written
+ private long numDirtyChunks; // number of incomplete compressed blocks written
/** Sole constructor. */
public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context,
String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk, int blockSize) throws IOException {
assert directory != null;
this.segment = si.name;
+ this.compressionMode = compressionMode;
this.compressor = compressionMode.newCompressor();
this.chunkSize = chunkSize;
this.maxDocsPerChunk = maxDocsPerChunk;
@@ -234,6 +242,7 @@ public final class CompressingStoredFiel
docBase += numBufferedDocs;
numBufferedDocs = 0;
bufferedDocs.length = 0;
+ numChunks++;
}
byte scratchBytes[] = new byte[16];
@@ -459,6 +468,7 @@ public final class CompressingStoredFiel
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numBufferedDocs > 0) {
flush();
+ numDirtyChunks++; // incomplete: we had to force this flush
} else {
assert bufferedDocs.length == 0;
}
@@ -466,9 +476,24 @@ public final class CompressingStoredFiel
throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
}
indexWriter.finish(numDocs, fieldsStream.getFilePointer());
+ fieldsStream.writeVLong(numChunks);
+ fieldsStream.writeVLong(numDirtyChunks);
CodecUtil.writeFooter(fieldsStream);
assert bufferedDocs.length == 0;
}
+
+ // bulk merge is scary: its caused corruption bugs in the past.
+ // we try to be extra safe with this impl, but add an escape hatch to
+ // have a workaround for undiscovered bugs.
+ static final String BULK_MERGE_ENABLED_SYSPROP = CompressingStoredFieldsWriter.class.getName() + ".enableBulkMerge";
+ static final boolean BULK_MERGE_ENABLED;
+ static {
+ boolean v = true;
+ try {
+ v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));
+ } catch (SecurityException ignored) {}
+ BULK_MERGE_ENABLED = v;
+ }
@Override
public int merge(MergeState mergeState) throws IOException {
@@ -491,8 +516,8 @@ public final class CompressingStoredFiel
final int maxDoc = mergeState.maxDocs[readerIndex];
final Bits liveDocs = mergeState.liveDocs[readerIndex];
- // if its some other format, or an older version of this format:
- if (matchingFieldsReader == null || matchingFieldsReader.getVersion() != VERSION_CURRENT) {
+ // if its some other format, or an older version of this format, or safety switch:
+ if (matchingFieldsReader == null || matchingFieldsReader.getVersion() != VERSION_CURRENT || BULK_MERGE_ENABLED == false) {
// naive merge...
StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[readerIndex];
if (storedFieldsReader != null) {
@@ -506,12 +531,80 @@ public final class CompressingStoredFiel
storedFieldsReader.visitDocument(docID, visitor);
finishDocument();
++docCount;
- mergeState.checkAbort.work(300);
}
+ } else if (matchingFieldsReader.getCompressionMode() == compressionMode &&
+ matchingFieldsReader.getChunkSize() == chunkSize &&
+ matchingFieldsReader.getPackedIntsVersion() == PackedInts.VERSION_CURRENT &&
+ liveDocs == null &&
+ !tooDirty(matchingFieldsReader)) {
+ // optimized merge, raw byte copy
+ // its not worth fine-graining this if there are deletions.
+
+ // if the format is older, its always handled by the naive merge case above
+ assert matchingFieldsReader.getVersion() == VERSION_CURRENT;
+ matchingFieldsReader.checkIntegrity();
+
+ // flush any pending chunks
+ if (numBufferedDocs > 0) {
+ flush();
+ numDirtyChunks++; // incomplete: we had to force this flush
+ }
+
+ // iterate over each chunk. we use the stored fields index to find chunk boundaries,
+ // read the docstart + doccount from the chunk header (we write a new header, since doc numbers will change),
+ // and just copy the bytes directly.
+ IndexInput rawDocs = matchingFieldsReader.getFieldsStream();
+ CompressingStoredFieldsIndexReader index = matchingFieldsReader.getIndexReader();
+ rawDocs.seek(index.getStartPointer(0));
+ int docID = 0;
+ while (docID < maxDoc) {
+ // read header
+ int base = rawDocs.readVInt();
+ if (base != docID) {
+ throw new CorruptIndexException("invalid state: base=" + base + ", docID=" + docID, rawDocs);
+ }
+ int code = rawDocs.readVInt();
+
+ // write a new index entry and new header for this chunk.
+ int bufferedDocs = code >>> 1;
+ indexWriter.writeIndex(bufferedDocs, fieldsStream.getFilePointer());
+ fieldsStream.writeVInt(docBase); // rebase
+ fieldsStream.writeVInt(code);
+ docID += bufferedDocs;
+ docBase += bufferedDocs;
+ docCount += bufferedDocs;
+
+ if (docID > maxDoc) {
+ throw new CorruptIndexException("invalid state: base=" + base + ", count=" + bufferedDocs + ", maxDoc=" + maxDoc, rawDocs);
+ }
+
+ // copy bytes until the next chunk boundary (or end of chunk data).
+ // using the stored fields index for this isn't the most efficient, but fast enough
+ // and is a source of redundancy for detecting bad things.
+ final long end;
+ if (docID == maxDoc) {
+ end = matchingFieldsReader.getMaxPointer();
+ } else {
+ end = index.getStartPointer(docID);
+ }
+ fieldsStream.copyBytes(rawDocs, end - rawDocs.getFilePointer());
+ }
+
+ if (rawDocs.getFilePointer() != matchingFieldsReader.getMaxPointer()) {
+ throw new CorruptIndexException("invalid state: pos=" + rawDocs.getFilePointer() + ", max=" + matchingFieldsReader.getMaxPointer(), rawDocs);
+ }
+
+ // since we bulk merged all chunks, we inherit any dirty ones from this segment.
+ numChunks += matchingFieldsReader.getNumChunks();
+ numDirtyChunks += matchingFieldsReader.getNumDirtyChunks();
} else {
// optimized merge, we copy serialized (but decompressed) bytes directly
// even on simple docs (1 stored field), it seems to help by about 20%
+
+ // if the format is older, its always handled by the naive merge case above
+ assert matchingFieldsReader.getVersion() == VERSION_CURRENT;
matchingFieldsReader.checkIntegrity();
+
for (int docID = 0; docID < maxDoc; docID++) {
if (liveDocs != null && liveDocs.get(docID) == false) {
continue;
@@ -522,11 +615,23 @@ public final class CompressingStoredFiel
numStoredFieldsInDoc = doc.numStoredFields;
finishDocument();
++docCount;
- mergeState.checkAbort.work(300);
}
}
}
finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
+
+ /**
+ * Returns true if we should recompress this reader, even though we could bulk merge compressed data
+ * <p>
+ * The last chunk written for a segment is typically incomplete, so without recompressing,
+ * in some worst-case situations (e.g. frequent reopen with tiny flushes), over time the
+ * compression ratio can degrade. This is a safety switch.
+ */
+ boolean tooDirty(CompressingStoredFieldsReader candidate) {
+ // more than 1% dirty, or more than hard limit of 1024 dirty chunks
+ return candidate.getNumDirtyChunks() > 1024 ||
+ candidate.getNumDirtyChunks() * 100 > candidate.getNumChunks();
+ }
}