You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2014/05/21 07:52:33 UTC
svn commit: r1596478 [1/3] - in /lucene/dev/branches/lucene5650: ./
dev-tools/ dev-tools/idea/lucene/spatial/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/scripts/ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/memory/...
Author: dweiss
Date: Wed May 21 05:52:32 2014
New Revision: 1596478
URL: http://svn.apache.org/r1596478
Log:
Merging with trunk.
Added:
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
- copied unchanged from r1596476, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
- copied unchanged from r1596476, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
- copied unchanged from r1596476, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
Removed:
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java
Modified:
lucene/dev/branches/lucene5650/ (props changed)
lucene/dev/branches/lucene5650/dev-tools/ (props changed)
lucene/dev/branches/lucene5650/dev-tools/idea/lucene/spatial/spatial.iml
lucene/dev/branches/lucene5650/dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml
lucene/dev/branches/lucene5650/dev-tools/scripts/smokeTestRelease.py
lucene/dev/branches/lucene5650/lucene/ (props changed)
lucene/dev/branches/lucene5650/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5650/lucene/codecs/ (props changed)
lucene/dev/branches/lucene5650/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
lucene/dev/branches/lucene5650/lucene/common-build.xml (contents, props changed)
lucene/dev/branches/lucene5650/lucene/core/ (props changed)
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java
lucene/dev/branches/lucene5650/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
lucene/dev/branches/lucene5650/lucene/misc/ (props changed)
lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
lucene/dev/branches/lucene5650/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java
lucene/dev/branches/lucene5650/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene5650/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/lucene5650/solr/ (props changed)
lucene/dev/branches/lucene5650/solr/CHANGES.txt
lucene/dev/branches/lucene5650/solr/core/ (props changed)
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/update/StreamingSolrServers.java
lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java
lucene/dev/branches/lucene5650/solr/core/src/test/org/apache/solr/request/TestFaceting.java
lucene/dev/branches/lucene5650/solr/solrj/ (props changed)
lucene/dev/branches/lucene5650/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/lucene5650/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
lucene/dev/branches/lucene5650/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
lucene/dev/branches/lucene5650/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/lucene5650/solr/test-framework/ (props changed)
lucene/dev/branches/lucene5650/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Modified: lucene/dev/branches/lucene5650/dev-tools/idea/lucene/spatial/spatial.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/dev-tools/idea/lucene/spatial/spatial.iml?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/dev-tools/idea/lucene/spatial/spatial.iml (original)
+++ lucene/dev/branches/lucene5650/dev-tools/idea/lucene/spatial/spatial.iml Wed May 21 05:52:32 2014
@@ -25,6 +25,7 @@
<orderEntry type="module" scope="TEST" module-name="lucene-test-framework" />
<orderEntry type="module" module-name="lucene-core" />
<orderEntry type="module" module-name="queries" />
+ <orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="analysis-common" scope="TEST"/>
</component>
</module>
Modified: lucene/dev/branches/lucene5650/dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml (original)
+++ lucene/dev/branches/lucene5650/dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml Wed May 21 05:52:32 2014
@@ -45,5 +45,6 @@
<orderEntry type="module" module-name="stempel" />
<orderEntry type="module" module-name="analysis-common" />
<orderEntry type="module" module-name="lucene-core" />
+ <orderEntry type="module" module-name="misc" />
</component>
</module>
Modified: lucene/dev/branches/lucene5650/dev-tools/scripts/smokeTestRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/dev-tools/scripts/smokeTestRelease.py?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/dev-tools/scripts/smokeTestRelease.py (original)
+++ lucene/dev/branches/lucene5650/dev-tools/scripts/smokeTestRelease.py Wed May 21 05:52:32 2014
@@ -468,12 +468,22 @@ def testChangesText(dir, version, projec
checkChangesContent(open(fullPath, encoding='UTF-8').read(), version, fullPath, project, False)
reChangesSectionHREF = re.compile('<a id="(.*?)".*?>(.*?)</a>', re.IGNORECASE)
-
+reUnderbarNotDashHTML = re.compile(r'<li>(\s*(LUCENE|SOLR)_\d\d\d\d+)')
+reUnderbarNotDashTXT = re.compile(r'\s+((LUCENE|SOLR)_\d\d\d\d+)', re.MULTILINE)
def checkChangesContent(s, version, name, project, isHTML):
if isHTML and s.find('Release %s' % version) == -1:
raise RuntimeError('did not see "Release %s" in %s' % (version, name))
+ if isHTML:
+ r = reUnderbarNotDashHTML
+ else:
+ r = reUnderbarNotDashTXT
+
+ m = r.search(s)
+ if m is not None:
+ raise RuntimeError('incorrect issue (_ instead of -) in %s: %s' % (name, m.group(1)))
+
if s.lower().find('not yet released') != -1:
raise RuntimeError('saw "not yet released" in %s' % name)
Modified: lucene/dev/branches/lucene5650/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/CHANGES.txt?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5650/lucene/CHANGES.txt Wed May 21 05:52:32 2014
@@ -80,6 +80,12 @@ API Changes
* LUCENE-4371: Removed IndexInputSlicer and Directory.createSlicer() and replaced
with IndexInput.slice(). (Robert Muir)
+* LUCENE-5678: IndexOutput no longer allows seeking, so it is no longer required
+ to use RandomAccessFile to write Indexes. Lucene now uses standard FileOutputStream
+ wrapped with OutputStreamIndexOutput to write index data. BufferedIndexOutput was
+ removed, because buffering and checksumming is provided by FilterOutputStreams,
+ provided by the JDK. (Uwe Schindler, Mike McCandless)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -148,6 +154,10 @@ API Changes
* LUCENE-5640: The Token class was deprecated. Since Lucene 2.9, TokenStreams
are using Attributes, Token is no longer used. (Uwe Schindler, Robert Muir)
+* LUCENE-5679: Consolidated IndexWriter.deleteDocuments(Term) and
+ IndexWriter.deleteDocuments(Query) with their varargs counterparts.
+ (Shai Erera)
+
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes
@@ -172,22 +182,57 @@ Optimizations
to 8 (for int/float) and 16 (for long/double), for faster indexing
time and smaller indices. (Robert Muir, Uwe Schindler, Mike McCandless)
+* LUCENE-5670: Add skip/FinalOutput to FST Outputs. (Christian
+ Ziech via Mike McCandless).
+
Bug fixes
-* LUCENE-5600: HttpClientBase did not properly consume a connection if a server
- error occurred. (Christoph Kaser via Shai Erera)
+* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws
+ a confusing OutOfMemoryError wrapped inside IOException if the FileChannel
+ mapping failed because of lack of virtual address space. The IOException is
+ rethrown with more useful information about the problem, omitting the
+ incorrect OutOfMemoryError. (Robert Muir, Uwe Schindler)
-* LUCENE-5559: Add additional argument validation for CapitalizationFilter
- and CodepointCountFilter. (Ahmet Arslan via Robert Muir)
+* LUCENE-5682: NPE in QueryRescorer when Scorer is null
+ (Joel Bernstein, Mike McCandless)
+
+* LUCENE-5691: DocTermOrds lookupTerm(BytesRef) would return incorrect results
+ if the underlying TermsEnum supports ord() and the insertion point would
+ be at the end. (Robert Muir)
+
+Test Framework
+
+* LUCENE-5622: Fail tests if they print over the given limit of bytes to
+ System.out or System.err. (Robert Muir, Dawid Weiss)
+
+* LUCENE-5619: Added backwards compatibility tests to ensure we can update existing
+ indexes with doc-values updates. (Shai Erera, Robert Muir)
+
+======================= Lucene 4.8.1 =======================
+
+Bug fixes
* LUCENE-5639: Fix PositionLengthAttribute implementation in Token class.
(Uwe Schindler, Robert Muir)
+* LUCENE-5635: IndexWriter didn't properly handle IOException on TokenStream.reset(),
+ which could leave the analyzer in an inconsistent state. (Robert Muir)
+
+* LUCENE-5599: HttpReplicator did not properly delegate bulk read() to wrapped
+ InputStream. (Christoph Kaser via Shai Erera)
+
+* LUCENE-5600: HttpClientBase did not properly consume a connection if a server
+ error occurred. (Christoph Kaser via Shai Erera)
+
* LUCENE-5628: Change getFiniteStrings to iterative not recursive
implementation, so that building suggesters on a long suggestion
doesn't risk overflowing the stack; previously it consumed one Java
- stack frame per character in the expanded suggestion (Robert Muir,
- Simon Willnauer, Mike McCandless).
+ stack frame per character in the expanded suggestion. If you are building
+ a suggester this is a nasty trap. (Robert Muir, Simon Willnauer,
+ Mike McCandless).
+
+* LUCENE-5559: Add additional argument validation for CapitalizationFilter
+ and CodepointCountFilter. (Ahmet Arslan via Robert Muir)
* LUCENE-5641: SimpleRateLimiter would silently rate limit at 8 MB/sec
even if you asked for higher rates. (Mike McCandless)
@@ -198,24 +243,28 @@ Bug fixes
many segments are flushed on a full-flush (e.g. to obtain a
near-real-time reader). (Simon Willnauer, Mike McCandless)
+* LUCENE-5653: JoinUtil with ScoreMode.Avg on a multi-valued field
+ with more than 256 values would throw exception.
+ (Mikhail Khludnev via Robert Muir)
+
+* LUCENE-5654: Fix various close() methods that could suppress
+ throwables such as OutOfMemoryError, instead returning scary messages
+ that look like index corruption. (Mike McCandless, Robert Muir)
+
+* LUCENE-5656: Fix rare fd leak in SegmentReader when multiple docvalues
+ fields have been updated with IndexWriter.updateXXXDocValue and one
+ hits exception. (Shai Erera, Robert Muir)
+
* LUCENE-5660: AnalyzingSuggester.build will now throw IllegalArgumentException if
you give it a longer suggestion than it can handle (Robert Muir, Mike McCandless)
-* LUCENE-5668: Fix off-by-one in TieredMergePolicy (Mike McCandless)
+* LUCENE-5662: Add missing checks to Field to prevent IndexWriter.abort
+ if a stored value is null. (Robert Muir)
-* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws
- a confusing OutOfMemoryError wrapped inside IOException if the FileChannel
- mapping failed because of lack of virtual address space. The IOException is
- rethrown with more useful information about the problem, omitting the
- incorrect OutOfMemoryError. (Robert Muir, Uwe Schindler)
-
-Test Framework
+* LUCENE-5668: Fix off-by-one in TieredMergePolicy (Mike McCandless)
-* LUCENE-5622: Fail tests if they print over the given limit of bytes to
- System.out or System.err. (Robert Muir, Dawid Weiss)
-
-* LUCENE-5619: Added backwards compatibility tests to ensure we can update existing
- indexes with doc-values updates. (Shai Erera, Robert Muir)
+* LUCENE-5671: Upgrade ICU version to fix an ICU concurrency problem that
+ could cause exceptions when indexing. (feedly team, Robert Muir)
* LUCENE-5650: Fail tests if they try to write to CWD, and make test temp dirs absolute.
(Ryan Ernst, Dawid Weiss)
Modified: lucene/dev/branches/lucene5650/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java Wed May 21 05:52:32 2014
@@ -299,6 +299,33 @@ class FSTTermOutputs extends Outputs<FST
}
return new TermData(longs, bytes, docFreq, totalTermFreq);
}
+
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ int bits = in.readByte() & 0xff;
+ int bit0 = bits & 1;
+ int bit1 = bits & 2;
+ int bit2 = bits & 4;
+ int bytesSize = (bits >>> 3);
+ if (bit1 > 0 && bytesSize == 0) { // determine extra length
+ bytesSize = in.readVInt();
+ }
+ if (bit0 > 0) { // not all-zero case
+ for (int pos = 0; pos < longsSize; pos++) {
+ in.readVLong();
+ }
+ }
+ if (bit1 > 0) { // bytes exists
+ in.skipBytes(bytesSize);
+ }
+ if (bit2 > 0) { // stats exist
+ int code = in.readVInt();
+ if (hasPos && (code & 1) == 0) {
+ in.readVLong();
+ }
+ }
+ }
@Override
public TermData getNoOutput() {
Modified: lucene/dev/branches/lucene5650/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/common-build.xml?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/common-build.xml (original)
+++ lucene/dev/branches/lucene5650/lucene/common-build.xml Wed May 21 05:52:32 2014
@@ -2202,7 +2202,7 @@ ${ant.project.name}.test.dependencies=${
<!-- GROOVY scripting engine for ANT tasks -->
<target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-configure">
- <ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.3.0"
+ <ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.3.1"
inline="true" conf="default" type="jar" transitive="true" pathid="groovy.classpath"/>
<taskdef name="groovy"
classname="org.codehaus.groovy.ant.Groovy"
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Wed May 21 05:52:32 2014
@@ -76,8 +76,8 @@ import org.apache.lucene.util.Version;
and otherwise open the existing index.</p>
<p>In either case, documents are added with {@link #addDocument(IndexDocument)
- addDocument} and removed with {@link #deleteDocuments(Term)} or {@link
- #deleteDocuments(Query)}. A document can be updated with {@link
+ addDocument} and removed with {@link #deleteDocuments(Term...)} or {@link
+ #deleteDocuments(Query...)}. A document can be updated with {@link
#updateDocument(Term, IndexDocument) updateDocument} (which just deletes
and then adds the entire document). When finished adding, deleting
and updating documents, {@link #close() close} should be called.</p>
@@ -1216,28 +1216,6 @@ public class IndexWriter implements Clos
}
}
- /**
- * Deletes the document(s) containing <code>term</code>.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
- *
- * @param term the term to identify the documents to be deleted
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public void deleteDocuments(Term term) throws IOException {
- ensureOpen();
- try {
- if (docWriter.deleteTerms(term)) {
- processEvents(true, false);
- }
- } catch (OutOfMemoryError oom) {
- handleOOM(oom, "deleteDocuments(Term)");
- }
- }
-
/** Expert: attempts to delete by document ID, as long as
* the provided reader is a near-real-time reader (from {@link
* DirectoryReader#open(IndexWriter,boolean)}). If the
@@ -1250,8 +1228,7 @@ public class IndexWriter implements Clos
* <b>NOTE</b>: this method can only delete documents
* visible to the currently open NRT reader. If you need
* to delete documents indexed after opening the NRT
- * reader you must use the other deleteDocument methods
- * (e.g., {@link #deleteDocuments(Term)}). */
+ * reader you must use {@link #deleteDocuments(Term...)}). */
public synchronized boolean tryDeleteDocument(IndexReader readerIn, int docID) throws IOException {
final AtomicReader reader;
@@ -1340,28 +1317,6 @@ public class IndexWriter implements Clos
}
/**
- * Deletes the document(s) matching the provided query.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
- *
- * @param query the query to identify the documents to be deleted
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public void deleteDocuments(Query query) throws IOException {
- ensureOpen();
- try {
- if (docWriter.deleteQueries(query)) {
- processEvents(true, false);
- }
- } catch (OutOfMemoryError oom) {
- handleOOM(oom, "deleteDocuments(Query)");
- }
- }
-
- /**
* Deletes the document(s) matching any of the provided queries.
* All given deletes are applied and flushed atomically at the same time.
*
@@ -2079,25 +2034,30 @@ public class IndexWriter implements Clos
/**
* Delete all documents in the index.
- *
- * <p>This method will drop all buffered documents and will
- * remove all segments from the index. This change will not be
- * visible until a {@link #commit()} has been called. This method
- * can be rolled back using {@link #rollback()}.</p>
- *
- * <p>NOTE: this method is much faster than using deleteDocuments( new MatchAllDocsQuery() ).
- * Yet, this method also has different semantics compared to {@link #deleteDocuments(Query)}
- * / {@link #deleteDocuments(Query...)} since internal data-structures are cleared as well
- * as all segment information is forcefully dropped anti-viral semantics like omitting norms
- * are reset or doc value types are cleared. Essentially a call to {@link #deleteAll()} is equivalent
- * to creating a new {@link IndexWriter} with {@link OpenMode#CREATE} which a delete query only marks
- * documents as deleted.</p>
- *
- * <p>NOTE: this method will forcefully abort all merges
- * in progress. If other threads are running {@link
- * #forceMerge}, {@link #addIndexes(IndexReader[])} or
- * {@link #forceMergeDeletes} methods, they may receive
- * {@link MergePolicy.MergeAbortedException}s.
+ *
+ * <p>
+ * This method will drop all buffered documents and will remove all segments
+ * from the index. This change will not be visible until a {@link #commit()}
+ * has been called. This method can be rolled back using {@link #rollback()}.
+ * </p>
+ *
+ * <p>
+ * NOTE: this method is much faster than using deleteDocuments( new
+ * MatchAllDocsQuery() ). Yet, this method also has different semantics
+ * compared to {@link #deleteDocuments(Query...)} since internal
+ * data-structures are cleared as well as all segment information is
+ * forcefully dropped anti-viral semantics like omitting norms are reset or
+ * doc value types are cleared. Essentially a call to {@link #deleteAll()} is
+ * equivalent to creating a new {@link IndexWriter} with
+ * {@link OpenMode#CREATE} which a delete query only marks documents as
+ * deleted.
+ * </p>
+ *
+ * <p>
+ * NOTE: this method will forcefully abort all merges in progress. If other
+ * threads are running {@link #forceMerge}, {@link #addIndexes(IndexReader[])}
+ * or {@link #forceMergeDeletes} methods, they may receive
+ * {@link MergePolicy.MergeAbortedException}s.
*/
public void deleteAll() throws IOException {
ensureOpen();
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java Wed May 21 05:52:32 2014
@@ -83,7 +83,7 @@ public class TrackingIndexWriter {
return indexingGen.get();
}
- /** Calls {@link IndexWriter#deleteDocuments(Term)} and
+ /** Calls {@link IndexWriter#deleteDocuments(Term...)} and
* returns the generation that reflects this change. */
public long deleteDocuments(Term t) throws IOException {
writer.deleteDocuments(t);
@@ -99,7 +99,7 @@ public class TrackingIndexWriter {
return indexingGen.get();
}
- /** Calls {@link IndexWriter#deleteDocuments(Query)} and
+ /** Calls {@link IndexWriter#deleteDocuments(Query...)} and
* returns the generation that reflects this change. */
public long deleteDocuments(Query q) throws IOException {
writer.deleteDocuments(q);
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java Wed May 21 05:52:32 2014
@@ -85,18 +85,23 @@ public abstract class QueryRescorer exte
scorer = weight.scorer(readerContext, null);
}
- int targetDoc = docID - docBase;
- int actualDoc = scorer.docID();
- if (actualDoc < targetDoc) {
- actualDoc = scorer.advance(targetDoc);
- }
+ if(scorer != null) {
+ int targetDoc = docID - docBase;
+ int actualDoc = scorer.docID();
+ if (actualDoc < targetDoc) {
+ actualDoc = scorer.advance(targetDoc);
+ }
- if (actualDoc == targetDoc) {
- // Query did match this doc:
- hit.score = combine(hit.score, true, scorer.score());
+ if (actualDoc == targetDoc) {
+ // Query did match this doc:
+ hit.score = combine(hit.score, true, scorer.score());
+ } else {
+ // Query did not match this doc:
+ assert actualDoc > targetDoc;
+ hit.score = combine(hit.score, false, 0.0f);
+ }
} else {
// Query did not match this doc:
- assert actualDoc > targetDoc;
hit.score = combine(hit.score, false, 0.0f);
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Wed May 21 05:52:32 2014
@@ -22,9 +22,10 @@ import org.apache.lucene.util.IOUtils;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.FilenameFilter;
+import java.io.FilterOutputStream;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -262,7 +263,7 @@ public abstract class FSDirectory extend
ensureOpen();
ensureCanWrite(name);
- return new FSIndexOutput(this, name);
+ return new FSIndexOutput(name);
}
protected void ensureCanWrite(String name) throws IOException {
@@ -338,58 +339,37 @@ public abstract class FSDirectory extend
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + getLockFactory();
}
- /**
- * Writes output with {@link RandomAccessFile#write(byte[], int, int)}
- */
- protected static class FSIndexOutput extends BufferedIndexOutput {
+ final class FSIndexOutput extends OutputStreamIndexOutput {
/**
- * The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
+ * The maximum chunk size is 8192 bytes, because {@link FileOutputStream} mallocs
* a native buffer outside of stack if the write buffer size is larger.
*/
- private static final int CHUNK_SIZE = 8192;
+ static final int CHUNK_SIZE = 8192;
- private final FSDirectory parent;
private final String name;
- private final RandomAccessFile file;
- private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
-
- public FSIndexOutput(FSDirectory parent, String name) throws IOException {
- super(CHUNK_SIZE);
- this.parent = parent;
- this.name = name;
- file = new RandomAccessFile(new File(parent.directory, name), "rw");
- isOpen = true;
- }
- @Override
- protected void flushBuffer(byte[] b, int offset, int size) throws IOException {
- assert isOpen;
- while (size > 0) {
- final int toWrite = Math.min(CHUNK_SIZE, size);
- file.write(b, offset, toWrite);
- offset += toWrite;
- size -= toWrite;
- }
- assert size == 0;
+ public FSIndexOutput(String name) throws IOException {
+ super(new FilterOutputStream(new FileOutputStream(new File(directory, name))) {
+ // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
+ @Override
+ public void write(byte[] b, int offset, int length) throws IOException {
+ while (length > 0) {
+ final int chunk = Math.min(length, CHUNK_SIZE);
+ out.write(b, offset, chunk);
+ length -= chunk;
+ offset += chunk;
+ }
+ }
+ }, CHUNK_SIZE);
+ this.name = name;
}
@Override
public void close() throws IOException {
- parent.onIndexOutputClosed(name);
- // only close the file if it has not been closed yet
- if (isOpen) {
- boolean success = false;
- try {
- super.close();
- success = true;
- } finally {
- isOpen = false;
- if (success) {
- IOUtils.close(file);
- } else {
- IOUtils.closeWhileHandlingException(file);
- }
- }
+ try {
+ onIndexOutputClosed(name);
+ } finally {
+ super.close();
}
}
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java Wed May 21 05:52:32 2014
@@ -1,4 +1,5 @@
package org.apache.lucene.store;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,6 +16,7 @@ package org.apache.lucene.store;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
import java.io.IOException;
/**
@@ -22,48 +24,58 @@ import java.io.IOException;
*
* @lucene.internal
*/
-final class RateLimitedIndexOutput extends BufferedIndexOutput {
+final class RateLimitedIndexOutput extends IndexOutput {
private final IndexOutput delegate;
- private final BufferedIndexOutput bufferedDelegate;
private final RateLimiter rateLimiter;
/** How many bytes we've written since we last called rateLimiter.pause. */
private long bytesSinceLastPause;
+
+ /** Cached here not not always have to call RateLimiter#getMinPauseCheckBytes()
+ * which does volatile read. */
+ private long currentMinPauseCheckBytes;
RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
- // TODO should we make buffer size configurable
- if (delegate instanceof BufferedIndexOutput) {
- bufferedDelegate = (BufferedIndexOutput) delegate;
- this.delegate = delegate;
- } else {
- this.delegate = delegate;
- bufferedDelegate = null;
- }
+ this.delegate = delegate;
this.rateLimiter = rateLimiter;
+ this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
}
@Override
- protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
- bytesSinceLastPause += len;
- if (bytesSinceLastPause > rateLimiter.getMinPauseCheckBytes()) {
- rateLimiter.pause(bytesSinceLastPause);
- bytesSinceLastPause = 0;
- }
- if (bufferedDelegate != null) {
- bufferedDelegate.flushBuffer(b, offset, len);
- } else {
- delegate.writeBytes(b, offset, len);
- }
-
+ public void close() throws IOException {
+ delegate.close();
}
@Override
- public void close() throws IOException {
- try {
- super.close();
- } finally {
- delegate.close();
- }
+ public long getFilePointer() {
+ return delegate.getFilePointer();
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ bytesSinceLastPause++;
+ checkRate();
+ delegate.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ bytesSinceLastPause += length;
+ checkRate();
+ delegate.writeBytes(b, offset, length);
+ }
+
+ private void checkRate() {
+ if (bytesSinceLastPause > currentMinPauseCheckBytes) {
+ rateLimiter.pause(bytesSinceLastPause);
+ bytesSinceLastPause = 0;
+ currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
+ }
}
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Wed May 21 05:52:32 2014
@@ -129,6 +129,14 @@ public final class ByteSequenceOutputs e
}
@Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ if (len != 0) {
+ in.skipBytes(len);
+ }
+ }
+
+ @Override
public BytesRef getNoOutput() {
return NO_OUTPUT;
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Wed May 21 05:52:32 2014
@@ -362,7 +362,7 @@ class BytesStore extends DataOutput {
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
setPosition(getPosition() + count);
}
@@ -430,7 +430,7 @@ class BytesStore extends DataOutput {
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
setPosition(getPosition() - count);
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java Wed May 21 05:52:32 2014
@@ -132,6 +132,14 @@ public final class CharSequenceOutputs e
return output;
}
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ for(int idx=0;idx<len;idx++) {
+ in.readVInt();
+ }
+ }
@Override
public CharsRef getNoOutput() {
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Wed May 21 05:52:32 2014
@@ -893,10 +893,10 @@ public final class FST<T> {
// skip this arc:
readLabel(in);
if (arc.flag(BIT_ARC_HAS_OUTPUT)) {
- outputs.read(in);
+ outputs.skipOutput(in);
}
if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) {
- outputs.readFinalOutput(in);
+ outputs.skipFinalOutput(in);
}
if (arc.flag(BIT_STOP_NODE)) {
} else if (arc.flag(BIT_TARGET_NEXT)) {
@@ -1252,11 +1252,11 @@ public final class FST<T> {
readLabel(in);
if (flag(flags, BIT_ARC_HAS_OUTPUT)) {
- outputs.read(in);
+ outputs.skipOutput(in);
}
if (flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) {
- outputs.readFinalOutput(in);
+ outputs.skipFinalOutput(in);
}
if (!flag(flags, BIT_STOP_NODE) && !flag(flags, BIT_TARGET_NEXT)) {
@@ -1330,9 +1330,6 @@ public final class FST<T> {
/** Returns true if this reader uses reversed bytes
* under-the-hood. */
public abstract boolean reversed();
-
- /** Skips bytes. */
- public abstract void skipBytes(int count);
}
private static class ArcAndState<T> {
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java Wed May 21 05:52:32 2014
@@ -41,7 +41,7 @@ final class ForwardBytesReader extends F
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
pos += count;
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Wed May 21 05:52:32 2014
@@ -131,6 +131,17 @@ public final class IntSequenceOutputs ex
return output;
}
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ if (len == 0) {
+ return;
+ }
+ for(int idx=0;idx<len;idx++) {
+ in.readVInt();
+ }
+ }
@Override
public IntsRef getNoOutput() {
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Wed May 21 05:52:32 2014
@@ -63,12 +63,25 @@ public abstract class Outputs<T> {
* #write(Object, DataOutput)}. */
public abstract T read(DataInput in) throws IOException;
+ /** Skip the output; defaults to just calling {@link #read}
+ * and discarding the result. */
+ public void skipOutput(DataInput in) throws IOException {
+ read(in);
+ }
+
/** Decode an output value previously written with {@link
* #writeFinalOutput(Object, DataOutput)}. By default this
* just calls {@link #read(DataInput)}. */
public T readFinalOutput(DataInput in) throws IOException {
return read(in);
}
+
+ /** Skip the output previously written with {@link #writeFinalOutput};
+ * defaults to just calling {@link #readFinalOutput} and discarding
+ * the result. */
+ public void skipFinalOutput(DataInput in) throws IOException {
+ skipOutput(in);
+ }
/** NOTE: this output is compared with == so you must
* ensure that all methods return the single object if
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Wed May 21 05:52:32 2014
@@ -148,6 +148,12 @@ public class PairOutputs<A,B> extends Ou
B output2 = outputs2.read(in);
return newPair(output1, output2);
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ outputs1.skipOutput(in);
+ outputs2.skipOutput(in);
+ }
@Override
public Pair<A,B> getNoOutput() {
Modified: lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java Wed May 21 05:52:32 2014
@@ -39,7 +39,7 @@ final class ReverseBytesReader extends F
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
pos -= count;
}
Modified: lucene/dev/branches/lucene5650/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java (original)
+++ lucene/dev/branches/lucene5650/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java Wed May 21 05:52:32 2014
@@ -107,6 +107,46 @@ public class TestQueryRescorer extends L
dir.close();
}
+ // Test LUCENE-5682
+ public void testNullScorerTermQuery() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ Document doc = new Document();
+ doc.add(newStringField("id", "0", Field.Store.YES));
+ doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newStringField("id", "1", Field.Store.YES));
+ // 1 extra token, but wizard and oz are close;
+ doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO));
+ w.addDocument(doc);
+ IndexReader r = w.getReader();
+ w.shutdown();
+
+ // Do ordinary BooleanQuery:
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(new TermQuery(new Term("field", "wizard")), Occur.SHOULD);
+ bq.add(new TermQuery(new Term("field", "oz")), Occur.SHOULD);
+ IndexSearcher searcher = getSearcher(r);
+ searcher.setSimilarity(new DefaultSimilarity());
+
+ TopDocs hits = searcher.search(bq, 10);
+ assertEquals(2, hits.totalHits);
+ assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id"));
+ assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id"));
+
+ // Now, resort using TermQuery on term that does not exist.
+ TermQuery tq = new TermQuery(new Term("field", "gold"));
+ TopDocs hits2 = QueryRescorer.rescore(searcher, hits, tq, 2.0, 10);
+
+ // Just testing that null scorer is handled.
+ assertEquals(2, hits2.totalHits);
+
+ r.close();
+ dir.close();
+ }
+
public void testCustomCombine() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Modified: lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java Wed May 21 05:52:32 2014
@@ -894,10 +894,15 @@ public class DocTermOrds {
@Override
public long lookupTerm(BytesRef key) {
try {
- if (te.seekCeil(key) == SeekStatus.FOUND) {
- return te.ord();
- } else {
- return -te.ord()-1;
+ switch (te.seekCeil(key)) {
+ case FOUND:
+ assert te.ord() >= 0;
+ return te.ord();
+ case NOT_FOUND:
+ assert te.ord() >= 0;
+ return -te.ord()-1;
+ default: /* END */
+ return -numTerms()-1;
}
} catch (IOException e) {
throw new RuntimeException(e);
Modified: lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java (original)
+++ lucene/dev/branches/lucene5650/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java Wed May 21 05:52:32 2014
@@ -122,6 +122,11 @@ public final class ListOfOutputs<T> exte
public Object read(DataInput in) throws IOException {
return outputs.read(in);
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ outputs.skipOutput(in);
+ }
@Override
public Object readFinalOutput(DataInput in) throws IOException {
@@ -136,6 +141,14 @@ public final class ListOfOutputs<T> exte
return outputList;
}
}
+
+ @Override
+ public void skipFinalOutput(DataInput in) throws IOException {
+ int count = in.readVInt();
+ for(int i=0;i<count;i++) {
+ outputs.skipOutput(in);
+ }
+ }
@Override
public Object getNoOutput() {
Modified: lucene/dev/branches/lucene5650/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java (original)
+++ lucene/dev/branches/lucene5650/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java Wed May 21 05:52:32 2014
@@ -579,6 +579,16 @@ public class TestDocTermOrds extends Luc
termsEnum.seekExact(2);
assertEquals("world", termsEnum.term().utf8ToString());
assertEquals(2, termsEnum.ord());
+
+ // lookupTerm(BytesRef)
+ assertEquals(-1, dv.lookupTerm(new BytesRef("apple")));
+ assertEquals(0, dv.lookupTerm(new BytesRef("beer")));
+ assertEquals(-2, dv.lookupTerm(new BytesRef("car")));
+ assertEquals(1, dv.lookupTerm(new BytesRef("hello")));
+ assertEquals(-3, dv.lookupTerm(new BytesRef("matter")));
+ assertEquals(2, dv.lookupTerm(new BytesRef("world")));
+ assertEquals(-4, dv.lookupTerm(new BytesRef("zany")));
+
ireader.close();
directory.close();
}
Modified: lucene/dev/branches/lucene5650/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene5650/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Wed May 21 05:52:32 2014
@@ -524,10 +524,6 @@ public class MockDirectoryWrapper extend
}
//System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
IndexOutput delegateOutput = in.createOutput(name, LuceneTestCase.newIOContext(randomState, context));
- if (randomState.nextInt(10) == 0){
- // once in a while wrap the IO in a Buffered IO with random buffer sizes
- delegateOutput = new BufferedIndexOutputWrapper(1+randomState.nextInt(BufferedIndexOutput.DEFAULT_BUFFER_SIZE), delegateOutput);
- }
final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
addFileHandle(io, name, Handle.Output);
openFilesForWrite.add(name);
@@ -947,29 +943,6 @@ public class MockDirectoryWrapper extend
in.copy(to, src, dest, context);
}
- final class BufferedIndexOutputWrapper extends BufferedIndexOutput {
- private final IndexOutput io;
-
- public BufferedIndexOutputWrapper(int bufferSize, IndexOutput io) {
- super(bufferSize);
- this.io = io;
- }
-
- @Override
- protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
- io.writeBytes(b, offset, len);
- }
-
- @Override
- public void close() throws IOException {
- try {
- super.close();
- } finally {
- io.close();
- }
- }
- }
-
/** Use this when throwing fake {@code IOException},
* e.g. from {@link MockDirectoryWrapper.Failure}. */
public static class FakeIOException extends IOException {
Modified: lucene/dev/branches/lucene5650/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/solr/CHANGES.txt?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5650/solr/CHANGES.txt Wed May 21 05:52:32 2014
@@ -82,9 +82,9 @@ Other Changes
* SOLR-5868: HttpClient should be configured to use ALLOW_ALL_HOSTNAME hostname
verifier to simplify SSL setup. (Steve Davids via Mark Miller)
-* SOLR-5495: Recovery strategy for leader partitioned from replica case. Hardening
- recovery scenarios after the leader receives an error trying to forward an
- update request to a replica. (Timothy Potter)
+* SOLR-5468: Allow a client application to request the minium achieved
+ replication factor for an update request (single or batch) by sending
+ an optional parameter "min_rf". (Timothy Potter)
================== 4.9.0 ==================
@@ -119,38 +119,10 @@ New Features
Bug Fixes
----------------------
-* SOLR-5904: ElectionContext can cancel an election when it should not if there
- was an exception while trying to register as the leader.
- (Mark Miller, Alan Woodward)
-
-* SOLR-5993: ZkController can warn about shard leader conflict even after the conflict
- is resolved. (Gregory Chanan via shalin)
-
* SOLR-6002: Fix a couple of ugly issues around SolrIndexWriter close and
rollback as well as how SolrIndexWriter manages it's ref counted directory
instance. (Mark Miller, Gregory Chanan)
-* SOLR-6017: Fix SimpleQParser to use query analyzer instead of index analyzer.
- (Ryan Ernst)
-
-* SOLR-6029: CollapsingQParserPlugin throws ArrayIndexOutOfBoundsException
- if elevated doc has been deleted from a segment. (Greg Harris, Joel Bernstein)
-
-* SOLR-6030: Use System.nanoTime() instead of currentTimeInMills() in LRUCache.warm.
- (Tomás Fernández Löbbe via shalin)
-
-* SOLR-6037: Fixed incorrect max/sum/stddev for Date fields in StatsComponent
- (Brett Lucey, hossman)
-
-* SOLR-6023: FieldAnalysisRequestHandler throws NPE if no parameters are supplied.
- (shalin)
-
-* SOLR-5090: SpellCheckComponent sometimes throws NPE if
- "spellcheck.alternativeTermCount" is set to zero (James Dyer).
-
-* SOLR-6039: fixed debug output when no results in response
- (Tomás Fernández Löbbe, hossman)
-
Other Changes
---------------------
@@ -180,8 +152,13 @@ Other Changes
* SOLR-5340: Add support for named snapshots (Varun Thacker via Noble Paul)
* LUCENE-5650: Tests can no longer write to CWD. Update log dir is now made relative
- to the instance dir if it is not an absolute path.
- (Ryan Ernst)
+ to the instance dir if it is not an absolute path. (Ryan Ernst, Dawid Weiss)
+
+* SOLR-5681: Make the processing of Collection API calls multi-threaded. (Anshum Gupta)
+
+* SOLR-5495: Recovery strategy for leader partitioned from replica case. Hardening
+ recovery scenarios after the leader receives an error trying to forward an
+ update request to a replica. (Timothy Potter)
Build
---------------------
@@ -192,24 +169,40 @@ Build
================== 4.8.1 ==================
-Versions of Major Components
----------------------
-Apache Tika 1.5
-Carrot2 3.9.0
-Velocity 1.7 and Velocity Tools 2.0
-Apache UIMA 2.3.1
-Apache ZooKeeper 3.4.6
-
Bug Fixes
----------------------
-* SOLR-6029: CollapsingQParserPlugin throws ArrayIndexOutOfBoundsException
+* SOLR-5904: ElectionContext can cancel an election when it should not if there
+ was an exception while trying to register as the leader.
+ (Mark Miller, Alan Woodward)
+
+* SOLR-5993: ZkController can warn about shard leader conflict even after the conflict
+ is resolved. (Gregory Chanan via shalin)
+
+* SOLR-6017: Fix SimpleQParser to use query analyzer
+ (Ryan Ernst)
+
+* SOLR_6029: CollapsingQParserPlugin throws ArrayIndexOutOfBoundsException
if elevated doc has been deleted from a segment. (Greg Harris, Joel Bernstein)
+* SOLR-6030: Use System.nanoTime() instead of currentTimeInMills() in LRUCache.warm.
+ (Tomás Fernández Löbbe via shalin)
+
+* SOLR-6037: Fixed incorrect max/sum/stddev for Date fields in StatsComponent
+ (Brett Lucey, hossman)
+
+* SOLR-6023: FieldAnalysisRequestHandler throws NPE if no parameters are supplied.
+ (shalin)
+
+* SOLR-5090: SpellCheckComponent sometimes throws NPE if
+ "spellcheck.alternativeTermCount" is set to zero (James Dyer).
+
+* SOLR-6039: fixed debug output when no results in response
+ (Tomás Fernández Löbbe, hossman)
+
* SOLR-6035: CloudSolrServer directUpdate routing should use getCoreUrl.
(Marvin Justice, Joel Bernstein)
-
================== 4.8.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java (original)
+++ lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java Wed May 21 05:52:32 2014
@@ -27,6 +27,7 @@ import org.apache.zookeeper.WatchedEvent
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -160,6 +161,12 @@ public class DistributedMap {
return zookeeper.exists(dir + "/" + prefix + trackingId, true);
}
+ public int size() throws KeeperException, InterruptedException {
+ Stat stat = new Stat();
+ zookeeper.getData(dir, null, stat, true);
+ return stat.getNumChildren();
+ }
+
public void remove(String trackingId) throws KeeperException, InterruptedException {
zookeeper.delete(dir + "/" + prefix + trackingId, -1, true);
}
Modified: lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java (original)
+++ lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java Wed May 21 05:52:32 2014
@@ -18,10 +18,6 @@
package org.apache.solr.cloud;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.TreeMap;
-
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -36,6 +32,12 @@ import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeMap;
+
/**
* A distributed queue from zk recipes.
*/
@@ -364,6 +366,103 @@ public class DistributedQueue {
}
}
+ public List<QueueEvent> peekTopN(int n, Set<String> excludeSet, Long wait)
+ throws KeeperException, InterruptedException {
+ ArrayList<QueueEvent> topN = new ArrayList<>();
+
+ LOG.debug("Peeking for top {} elements. ExcludeSet: " + excludeSet.toString());
+ boolean waitedEnough = false;
+ TimerContext time = null;
+ if (wait == Long.MAX_VALUE) time = stats.time(dir + "_peekTopN_wait_forever");
+ else time = stats.time(dir + "_peekTopN_wait" + wait);
+
+ try {
+ TreeMap<Long, String> orderedChildren;
+ while (true) {
+ LatchChildWatcher childWatcher = new LatchChildWatcher();
+ try {
+ orderedChildren = orderedChildren(childWatcher);
+ } catch (KeeperException.NoNodeException e) {
+ zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true);
+ continue;
+ }
+
+ if (orderedChildren.size() == 0) {
+ if(waitedEnough) return null;
+ childWatcher.await(wait == Long.MAX_VALUE ? DEFAULT_TIMEOUT : wait);
+ waitedEnough = wait != Long.MAX_VALUE;
+ continue;
+ }
+
+ for (String headNode : orderedChildren.values()) {
+ if (headNode != null && topN.size() < n) {
+ try {
+ String id = dir + "/" + headNode;
+ if (excludeSet != null && excludeSet.contains(id)) continue;
+ QueueEvent queueEvent = new QueueEvent(id,
+ zookeeper.getData(dir + "/" + headNode, null, null, true), null);
+ topN.add(queueEvent);
+ } catch (KeeperException.NoNodeException e) {
+ // Another client removed the node first, try next
+ }
+ } else {
+ if (topN.size() >= 1) {
+ printQueueEventsListElementIds(topN);
+ return topN;
+ }
+ }
+ }
+
+ if (topN.size() > 0 ) {
+ printQueueEventsListElementIds(topN);
+ return topN;
+ }
+ if (waitedEnough) {
+ LOG.debug("Waited enough, returning null after peekTopN");
+ return null;
+ }
+ childWatcher.await(wait == Long.MAX_VALUE ? DEFAULT_TIMEOUT : wait);
+ waitedEnough = wait != Long.MAX_VALUE;
+ }
+ } finally {
+ time.stop();
+ }
+ }
+
+ private void printQueueEventsListElementIds(ArrayList<QueueEvent> topN) {
+ if(LOG.isDebugEnabled()) {
+ StringBuffer sb = new StringBuffer("[");
+ for(QueueEvent queueEvent: topN) {
+ sb.append(queueEvent.getId()).append(", ");
+ }
+ sb.append("]");
+ LOG.debug("Returning topN elements: {}", sb.toString());
+ }
+ }
+
+
+ /**
+ *
+ * Gets last element of the Queue without removing it.
+ */
+ public String getTailId() throws KeeperException, InterruptedException {
+ TreeMap<Long, String> orderedChildren = null;
+ orderedChildren = orderedChildren(null);
+ if(orderedChildren == null || orderedChildren.isEmpty()) return null;
+
+ for(String headNode : orderedChildren.descendingMap().values())
+ if (headNode != null) {
+ try {
+ QueueEvent queueEvent = new QueueEvent(dir + "/" + headNode, zookeeper.getData(dir + "/" + headNode,
+ null, null, true), null);
+ return queueEvent.getId();
+ } catch (KeeperException.NoNodeException e) {
+ // Another client removed the node first, try next
+ }
+ }
+ return null;
+ }
+
public static class QueueEvent {
@Override
public int hashCode() {
Modified: lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Wed May 21 05:52:32 2014
@@ -321,7 +321,14 @@ final class ShardLeaderElectionContext e
try (SolrCore core = cc.getCore(coreName)) {
CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
String coll = cloudDesc.getCollectionName();
- String shardId = cloudDesc.getShardId();
+ String shardId = cloudDesc.getShardId();
+
+ if (coll == null || shardId == null) {
+ log.error("Cannot start leader-initiated recovery on new leader (core="+
+ coreName+") because collection and/or shard is null!");
+ return;
+ }
+
String znodePath = zkController.getLeaderInitiatedRecoveryZnodePath(coll, shardId);
List<String> replicas = null;
try {
Modified: lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1596478&r1=1596477&r2=1596478&view=diff
==============================================================================
--- lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5650/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed May 21 05:52:32 2014
@@ -1200,7 +1200,6 @@ public class Overseer {
}
}
} finally {
-
if (ccThread != null) {
try {
ccThread.close();
@@ -1309,16 +1308,19 @@ public class Overseer {
public void success(String operation) {
String op = operation.toLowerCase(Locale.ROOT);
- Stat stat = stats.get(op);
- if (stat == null) {
- stat = new Stat();
- stats.put(op, stat);
+ synchronized (stats) {
+ Stat stat = stats.get(op);
+ if (stat == null) {
+ stat = new Stat();
+ stats.put(op, stat);
+ }
+ stat.success.incrementAndGet();
}
- stat.success.incrementAndGet();
}
public void error(String operation) {
String op = operation.toLowerCase(Locale.ROOT);
+ synchronized (stats) {
Stat stat = stats.get(op);
if (stat == null) {
stat = new Stat();
@@ -1326,20 +1328,26 @@ public class Overseer {
}
stat.errors.incrementAndGet();
}
+ }
public TimerContext time(String operation) {
String op = operation.toLowerCase(Locale.ROOT);
- Stat stat = stats.get(op);
+ Stat stat;
+ synchronized (stats) {
+ stat = stats.get(op);
if (stat == null) {
stat = new Stat();
stats.put(op, stat);
}
+ }
return stat.requestTime.time();
}
public void storeFailureDetails(String operation, ZkNodeProps request, SolrResponse resp) {
String op = operation.toLowerCase(Locale.ROOT);
- Stat stat = stats.get(op);
+ Stat stat ;
+ synchronized (stats) {
+ stat = stats.get(op);
if (stat == null) {
stat = new Stat();
stats.put(op, stat);
@@ -1352,6 +1360,7 @@ public class Overseer {
failedOps.addLast(new FailedOp(request, resp));
}
}
+ }
public List<FailedOp> getFailureDetails(String operation) {
Stat stat = stats.get(operation.toLowerCase(Locale.ROOT));