You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/04 23:26:15 UTC
svn commit: r1369470 [20/23] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/tartarus/snowball/ext/ luce...
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Sat Aug 4 21:26:10 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -116,4 +117,21 @@ public class TestSnowball extends BaseTo
checkOneTermReuse(a, "", "");
}
}
+
+ public void testRandomStrings() throws IOException {
+ for (String lang : SNOWBALL_LANGS) {
+ checkRandomStrings(lang);
+ }
+ }
+
+ public void checkRandomStrings(final String snowballLanguage) throws IOException {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer t = new MockTokenizer(reader);
+ return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
+ }
+ };
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java Sat Aug 4 21:26:10 2012
@@ -331,7 +331,9 @@ public class TaskSequence extends PerfTa
// Forwards top request to children
if (runningParallelTasks != null) {
for(ParallelTask t : runningParallelTasks) {
- t.task.stopNow();
+ if (t != null) {
+ t.task.stopNow();
+ }
}
}
}
@@ -355,6 +357,12 @@ public class TaskSequence extends PerfTa
// run threads
startThreads(t);
+ if (stopNow) {
+ for (ParallelTask task : t) {
+ task.task.stopNow();
+ }
+ }
+
// wait for all threads to complete
int count = 0;
for (int i = 0; i < t.length; i++) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/build.xml Sat Aug 4 21:26:10 2012
@@ -35,10 +35,9 @@
MIGRATE.txt,JRE_VERSION_MIGRATION.txt,
CHANGES.txt,
**/lib/*.jar,
- **/lib/*LICENSE*.txt,
- **/lib/*NOTICE*.txt,
+ licenses/**,
*/docs/,**/README*"
- excludes="build/**,site/**"
+ excludes="build/**,site/**,tools/**"
/>
@@ -572,4 +571,8 @@
</sequential>
</target>
+ <target name="jar-checksums" depends="clean-jars,resolve">
+ <jar-checksum-macro srcdir="${common.dir}" dstdir="${common.dir}/licenses"/>
+ </target>
+
</project>
Modified: lucene/dev/branches/pforcodec_3892/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/common-build.xml?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/common-build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/common-build.xml Sat Aug 4 21:26:10 2012
@@ -1567,6 +1567,43 @@ ${tests-output}/junit4-*.suites - pe
</sequential>
</macrodef>
+ <macrodef name="jar-checksum-macro">
+ <attribute name="srcdir"/>
+ <attribute name="dstdir"/>
+ <sequential>
+ <delete>
+ <fileset dir="@{dstdir}">
+ <include name="**/*.jar.sha1"/>
+ </fileset>
+ </delete>
+
+ <!-- checksum task does not have a flatten=true -->
+ <tempfile property="jar-checksum.temp.dir"/>
+ <mkdir dir="${jar-checksum.temp.dir}"/>
+ <copy todir="${jar-checksum.temp.dir}" flatten="true">
+ <fileset dir="@{srcdir}">
+ <include name="**/*.jar"/>
+ <!-- todo make this something passed into the macro and not some hardcoded set -->
+ <exclude name="build/**"/>
+ <exclude name="dist/**"/>
+ <exclude name="package/**"/>
+ <exclude name="example/exampledocs/**"/>
+ </fileset>
+ </copy>
+
+ <checksum algorithm="SHA1" fileext=".sha1" todir="@{dstdir}">
+ <fileset dir="${jar-checksum.temp.dir}"/>
+ </checksum>
+
+ <delete dir="${jar-checksum.temp.dir}"/>
+
+ <fixcrlf
+ srcdir="@{dstdir}"
+ includes="**/*.jar.sha1"
+ eol="lf" fixlast="true" encoding="US-ASCII" />
+ </sequential>
+ </macrodef>
+
<macrodef name="sign-artifacts-macro">
<attribute name="artifacts.dir"/>
<sequential>
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java Sat Aug 4 21:26:10 2012
@@ -53,7 +53,13 @@ public abstract class PostingsFormat imp
/** Reads a segment. NOTE: by the time this call
* returns, it must hold open any files it will need to
- * use; else, those files may be deleted. */
+ * use; else, those files may be deleted.
+ * Additionally, required files may be deleted during the execution of
+ * this call before there is a chance to open them. Under these
+ * circumstances an IOException should be thrown by the implementation.
+ * IOExceptions are expected and will automatically cause a retry of the
+ * segment opening logic with the newly revised segments.
+ * */
public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/Field.java Sat Aug 4 21:26:10 2012
@@ -377,6 +377,11 @@ public class Field implements IndexableF
* @see org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float)
*/
public void setBoost(float boost) {
+ if (boost != 1.0f) {
+ if (type.indexed() == false || type.omitNorms()) {
+ throw new IllegalArgumentException("You cannot set an index-time boost on an unindexed field, or one that omits norms");
+ }
+ }
this.boost = boost;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StoredField.java Sat Aug 4 21:26:10 2012
@@ -49,6 +49,7 @@ public final class StoredField extends F
super(name, value, TYPE);
}
+ // TODO: not great but maybe not a big problem?
public StoredField(String name, int value) {
super(name, TYPE);
fieldsData = value;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/StringField.java Sat Aug 4 21:26:10 2012
@@ -54,9 +54,4 @@ public final class StringField extends F
public StringField(String name, String value, Store stored) {
super(name, value, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
}
-
- @Override
- public String stringValue() {
- return (fieldsData == null) ? null : fieldsData.toString();
- }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/document/TextField.java Sat Aug 4 21:26:10 2012
@@ -46,9 +46,9 @@ public final class TextField extends Fie
// TODO: add sugar for term vectors...?
- /** Creates a new TextField with Reader value. */
- public TextField(String name, Reader reader, Store store) {
- super(name, reader, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
+ /** Creates a new un-stored TextField with Reader value. */
+ public TextField(String name, Reader reader) {
+ super(name, reader, TYPE_NOT_STORED);
}
/** Creates a new TextField with String value. */
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java Sat Aug 4 21:26:10 2012
@@ -1,8 +1,5 @@
package org.apache.lucene.index;
-import java.util.Collections;
-import java.util.List;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -20,6 +17,9 @@ import java.util.List;
* limitations under the License.
*/
+import java.util.Collections;
+import java.util.List;
+
/**
* {@link IndexReaderContext} for {@link AtomicReader} instances
* @lucene.experimental
@@ -51,8 +51,9 @@ public final class AtomicReaderContext e
@Override
public List<AtomicReaderContext> leaves() {
- if (!isTopLevel)
+ if (!isTopLevel) {
throw new UnsupportedOperationException("This is not a top-level context.");
+ }
assert leaves != null;
return leaves;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Sat Aug 4 21:26:10 2012
@@ -75,6 +75,9 @@ public abstract class DirectoryReader ex
* memory usage, at the expense of higher latency when
* loading a TermInfo. The default value is 1. Set this
* to -1 to skip loading the terms index entirely.
+ * <b>NOTE:</b> divisor settings > 1 do not apply to all PostingsFormat
+ * implementations, including the default one in this release. It only makes
+ * sense for terms indexes that can efficiently re-sample terms at load time.
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws IOException {
@@ -126,6 +129,9 @@ public abstract class DirectoryReader ex
* memory usage, at the expense of higher latency when
* loading a TermInfo. The default value is 1. Set this
* to -1 to skip loading the terms index entirely.
+ * <b>NOTE:</b> divisor settings > 1 do not apply to all PostingsFormat
+ * implementations, including the default one in this release. It only makes
+ * sense for terms indexes that can efficiently re-sample terms at load time.
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Aug 4 21:26:10 2012
@@ -202,11 +202,9 @@ final class DocumentsWriter {
* discarding any docs added since last flush. */
synchronized void abort() {
boolean success = false;
- synchronized (this) {
- deleteQueue.clear();
- }
try {
+ deleteQueue.clear();
if (infoStream.isEnabled("DW")) {
infoStream.message("DW", "abort");
}
@@ -230,6 +228,7 @@ final class DocumentsWriter {
perThread.unlock();
}
}
+ flushControl.abortPendingFlushes();
flushControl.waitForFlush();
success = true;
} finally {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Sat Aug 4 21:26:10 2012
@@ -567,19 +567,34 @@ final class DocumentsWriterFlushControl
}
synchronized void abortFullFlushes() {
+ try {
+ abortPendingFlushes();
+ } finally {
+ fullFlush = false;
+ }
+ }
+
+ synchronized void abortPendingFlushes() {
try {
for (DocumentsWriterPerThread dwpt : flushQueue) {
- doAfterFlush(dwpt);
- dwpt.abort();
+ try {
+ dwpt.abort();
+ doAfterFlush(dwpt);
+ } catch (Throwable ex) {
+ // ignore - keep on aborting the flush queue
+ }
}
for (BlockedFlush blockedFlush : blockedFlushes) {
- flushingWriters
- .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes));
- doAfterFlush(blockedFlush.dwpt);
- blockedFlush.dwpt.abort();
+ try {
+ flushingWriters
+ .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes));
+ blockedFlush.dwpt.abort();
+ doAfterFlush(blockedFlush.dwpt);
+ } catch (Throwable ex) {
+ // ignore - keep on aborting the blocked queue
+ }
}
} finally {
- fullFlush = false;
flushQueue.clear();
blockedFlushes.clear();
updateStallState();
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java Sat Aug 4 21:26:10 2012
@@ -1,7 +1,5 @@
package org.apache.lucene.index;
-import java.util.List;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,6 +17,8 @@ import java.util.List;
* limitations under the License.
*/
+import java.util.List;
+
/**
* A struct like class that represents a hierarchical relationship between
* {@link IndexReader} instances.
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Aug 4 21:26:10 2012
@@ -1241,6 +1241,78 @@ public class IndexWriter implements Clos
}
}
+ /** Expert: attempts to delete by document ID, as long as
+ * the provided reader is a near-real-time reader (from {@link
+ * DirectoryReader#open(IndexWriter,boolean)}). If the
+ * provided reader is an NRT reader obtained from this
+ * writer, and its segment has not been merged away, then
+ * the delete succeeds and this method returns true; else, it
+ * returns false the caller must then separately delete by
+ * Term or Query.
+ *
+ * <b>NOTE</b>: this method can only delete documents
+ * visible to the currently open NRT reader. If you need
+ * to delete documents indexed after opening the NRT
+ * reader you must use the other deleteDocument methods
+ * (e.g., {@link #deleteDocuments(Term)}). */
+ public synchronized boolean tryDeleteDocument(IndexReader readerIn, int docID) throws IOException {
+
+ final AtomicReader reader;
+ if (readerIn instanceof AtomicReader) {
+ // Reader is already atomic: use the incoming docID:
+ reader = (AtomicReader) readerIn;
+ } else {
+ // Composite reader: lookup sub-reader and re-base docID:
+ List<AtomicReaderContext> leaves = readerIn.getTopReaderContext().leaves();
+ int subIndex = ReaderUtil.subIndex(docID, leaves);
+ reader = leaves.get(subIndex).reader();
+ docID -= leaves.get(subIndex).docBase;
+ assert docID >= 0;
+ assert docID < reader.maxDoc();
+ }
+
+ if (!(reader instanceof SegmentReader)) {
+ throw new IllegalArgumentException("the reader must be a SegmentReader or composite reader containing only SegmentReaders");
+ }
+
+ final SegmentInfoPerCommit info = ((SegmentReader) reader).getSegmentInfo();
+
+ // TODO: this is a slow linear search, but, number of
+ // segments should be contained unless something is
+ // seriously wrong w/ the index, so it should be a minor
+ // cost:
+
+ if (segmentInfos.indexOf(info) != -1) {
+ ReadersAndLiveDocs rld = readerPool.get(info, false);
+ if (rld != null) {
+ synchronized(bufferedDeletesStream) {
+ rld.initWritableLiveDocs();
+ if (rld.delete(docID)) {
+ final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
+ if (fullDelCount == rld.info.info.getDocCount()) {
+ // If a merge has already registered for this
+ // segment, we leave it in the readerPool; the
+ // merge will skip merging it and will then drop
+ // it once it's done:
+ if (!mergingSegments.contains(rld.info)) {
+ segmentInfos.remove(rld.info);
+ readerPool.drop(rld.info);
+ checkpoint();
+ }
+ }
+ }
+ //System.out.println(" yes " + info.info.name + " " + docID);
+ return true;
+ }
+ } else {
+ //System.out.println(" no rld " + info.info.name + " " + docID);
+ }
+ } else {
+ //System.out.println(" no seg " + info.info.name + " " + docID);
+ }
+ return false;
+ }
+
/**
* Deletes the document(s) containing any of the
* terms. All given deletes are applied and flushed atomically
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java Sat Aug 4 21:26:10 2012
@@ -392,7 +392,7 @@ public final class IndexWriterConfig ext
* @see #setMaxBufferedDocs(int)
* @see #setRAMBufferSizeMB(double)
*/
- public IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) {
+ IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) {
this.flushPolicy = flushPolicy;
return this;
}
@@ -422,7 +422,7 @@ public final class IndexWriterConfig ext
}
@Override
- public FlushPolicy getFlushPolicy() {
+ FlushPolicy getFlushPolicy() {
return flushPolicy;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Sat Aug 4 21:26:10 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; // javadocs
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -146,6 +147,29 @@ public class LiveIndexWriterConfig {
* Takes effect immediately, but only applies to newly flushed/merged
* segments.
*
+ * <p>
+ * <b>NOTE:</b> This parameter does not apply to all PostingsFormat implementations,
+ * including the default one in this release. It only makes sense for term indexes
+ * that are implemented as a fixed gap between terms. For example,
+ * {@link Lucene40PostingsFormat} implements the term index instead based upon how
+ * terms share prefixes. To configure its parameters (the minimum and maximum size
+ * for a block), you would instead use {@link Lucene40PostingsFormat#Lucene40PostingsFormat(int, int)}.
+ * which can also be configured on a per-field basis:
+ * <pre class="prettyprint">
+ * //customize Lucene40PostingsFormat, passing minBlockSize=50, maxBlockSize=100
+ * final PostingsFormat tweakedPostings = new Lucene40PostingsFormat(50, 100);
+ * iwc.setCodec(new Lucene40Codec() {
+ * @Override
+ * public PostingsFormat getPostingsFormatForField(String field) {
+ * if (field.equals("fieldWithTonsOfTerms"))
+ * return tweakedPostings;
+ * else
+ * return super.getPostingsFormatForField(field);
+ * }
+ * });
+ * </pre>
+ * Note that other implementations may have their own parameters, or no parameters at all.
+ *
* @see IndexWriterConfig#DEFAULT_TERM_INDEX_INTERVAL
*/
public LiveIndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here
@@ -335,6 +359,10 @@ public class LiveIndexWriterConfig {
* <p>
* Takes effect immediately, but only applies to readers opened after this
* call
+ * <p>
+ * <b>NOTE:</b> divisor settings > 1 do not apply to all PostingsFormat
+ * implementations, including the default one in this release. It only makes
+ * sense for terms indexes that can efficiently re-sample terms at load time.
*/
public LiveIndexWriterConfig setReaderTermsIndexDivisor(int divisor) {
if (divisor <= 0 && divisor != -1) {
@@ -462,7 +490,7 @@ public class LiveIndexWriterConfig {
/**
* @see IndexWriterConfig#setFlushPolicy(FlushPolicy)
*/
- public FlushPolicy getFlushPolicy() {
+ FlushPolicy getFlushPolicy() {
return flushPolicy;
}
@@ -497,7 +525,6 @@ public class LiveIndexWriterConfig {
sb.append("mergePolicy=").append(getMergePolicy()).append("\n");
sb.append("indexerThreadPool=").append(getIndexerThreadPool()).append("\n");
sb.append("readerPooling=").append(getReaderPooling()).append("\n");
- sb.append("flushPolicy=").append(getFlushPolicy()).append("\n");
sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n");
return sb.toString();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Sat Aug 4 21:26:10 2012
@@ -122,7 +122,7 @@ public class FuzzyTermsEnum extends Term
this.realPrefixLength = prefixLength > termLength ? termLength : prefixLength;
// if minSimilarity >= 1, we treat it as number of edits
if (minSimilarity >= 1f) {
- this.minSimilarity = 1 - (minSimilarity+1) / this.termLength;
+ this.minSimilarity = 0; // just driven by number of edits
maxEdits = (int) minSimilarity;
raw = true;
} else {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/NRTManager.java Sat Aug 4 21:26:10 2012
@@ -27,6 +27,7 @@ import java.util.concurrent.locks.Reentr
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.SegmentInfoPerCommit;
import org.apache.lucene.index.IndexReader; // javadocs
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
@@ -254,6 +255,14 @@ public class NRTManager extends Referenc
long getAndIncrementGeneration() {
return indexingGen.getAndIncrement();
}
+
+ public long tryDeleteDocument(IndexReader reader, int docID) throws IOException {
+ if (writer.tryDeleteDocument(reader, docID)) {
+ return indexingGen.get();
+ } else {
+ return -1;
+ }
+ }
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sat Aug 4 21:26:10 2012
@@ -54,6 +54,11 @@ public final class FixedBitSet extends D
bits = new long[bits2words(numBits)];
}
+ public FixedBitSet(long[]storedBits,int numBits) {
+ this.numBits = numBits;
+ this.bits = storedBits;
+ }
+
/** Makes full copy. */
public FixedBitSet(FixedBitSet other) {
bits = new long[other.bits.length];
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug 4 21:26:10 2012
@@ -18,6 +18,7 @@ org.apache.lucene.codecs.pulsing.Pulsing
org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
org.apache.lucene.codecs.memory.MemoryPostingsFormat
org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat
+org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat
+org.apache.lucene.codecs.memory.DirectPostingsFormat
org.apache.lucene.codecs.block.BlockPostingsFormat
org.apache.lucene.codecs.blockpacked.BlockPackedPostingsFormat
-org.apache.lucene.codecs.memory.DirectPostingsFormat
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Sat Aug 4 21:26:10 2012
@@ -290,33 +290,4 @@ public class TestDocument extends Lucene
// expected
}
}
-
- public void testBoost() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- IndexWriter iw = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new StringField("field1", "sometext", Field.Store.YES));
- doc.add(new TextField("field2", "sometext", Field.Store.NO));
- doc.add(new StringField("foo", "bar", Field.Store.NO));
- iw.addDocument(doc); // add an 'ok' document
- try {
- doc = new Document();
- // try to boost with norms omitted
- StringField field = new StringField("foo", "baz", Field.Store.NO);
- field.setBoost(5.0f);
- doc.add(field);
- iw.addDocument(doc);
- fail("didn't get any exception, boost silently discarded");
- } catch (UnsupportedOperationException expected) {
- // expected
- }
- DirectoryReader ir = DirectoryReader.open(iw, false);
- assertEquals(1, ir.numDocs());
- assertEquals("sometext", ir.document(0).get("field1"));
- ir.close();
- iw.close();
- dir.close();
- }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestByteSlices.java Sat Aug 4 21:26:10 2012
@@ -40,24 +40,41 @@ public class TestByteSlices extends Luce
counters[stream] = 0;
}
- int num = atLeast(10000);
+ int num = atLeast(3000);
for (int iter = 0; iter < num; iter++) {
- int stream = random().nextInt(NUM_STREAM);
- if (VERBOSE)
+ int stream;
+ if (random().nextBoolean()) {
+ stream = random().nextInt(3);
+ } else {
+ stream = random().nextInt(NUM_STREAM);
+ }
+
+ if (VERBOSE) {
System.out.println("write stream=" + stream);
+ }
if (starts[stream] == -1) {
final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.byteOffset;
- if (VERBOSE)
+ if (VERBOSE) {
System.out.println(" init to " + starts[stream]);
+ }
}
writer.init(uptos[stream]);
- int numValue = random().nextInt(20);
+ int numValue;
+ if (random().nextInt(10) == 3) {
+ numValue = random().nextInt(100);
+ } else if (random().nextInt(5) == 3) {
+ numValue = random().nextInt(3);
+ } else {
+ numValue = random().nextInt(20);
+ }
+
for(int j=0;j<numValue;j++) {
- if (VERBOSE)
+ if (VERBOSE) {
System.out.println(" write " + (counters[stream]+j));
+ }
// write some large (incl. negative) ints:
writer.writeVInt(random().nextInt());
writer.writeVInt(counters[stream]+j);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Sat Aug 4 21:26:10 2012
@@ -186,7 +186,7 @@ public class TestDoc extends LuceneTestC
File file = new File(workDir, fileName);
Document doc = new Document();
InputStreamReader is = new InputStreamReader(new FileInputStream(file), "UTF-8");
- doc.add(new TextField("contents", is, Field.Store.NO));
+ doc.add(new TextField("contents", is));
writer.addDocument(doc);
writer.commit();
is.close();
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat Aug 4 21:26:10 2012
@@ -891,9 +891,12 @@ public class TestIndexWriterDelete exten
}
public void testIndexingThenDeleting() throws Exception {
+ // TODO: move this test to its own class and just @SuppressCodecs?
+ // TODO: is it enough to just use newFSDirectory?
final String fieldFormat = _TestUtil.getPostingsFormat("field");
assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory"));
assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText"));
+ assumeFalse("This test cannot run with Direct codec", fieldFormat.equals("Direct"));
final Random r = random();
Directory dir = newDirectory();
// note this test explicitly disables payloads
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sat Aug 4 21:26:10 2012
@@ -1542,4 +1542,73 @@ public class TestIndexWriterExceptions e
iw.close();
dir.close();
}
+
+ public void testBoostOmitNorms() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new StringField("field1", "sometext", Field.Store.YES));
+ doc.add(new TextField("field2", "sometext", Field.Store.NO));
+ doc.add(new StringField("foo", "bar", Field.Store.NO));
+ iw.addDocument(doc); // add an 'ok' document
+ try {
+ doc = new Document();
+ // try to boost with norms omitted
+ List<IndexableField> list = new ArrayList<IndexableField>();
+ list.add(new IndexableField() {
+
+ @Override
+ public String name() {
+ return "foo";
+ }
+
+ @Override
+ public IndexableFieldType fieldType() {
+ return StringField.TYPE_NOT_STORED;
+ }
+
+ @Override
+ public float boost() {
+ return 5f;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return null;
+ }
+
+ @Override
+ public String stringValue() {
+ return "baz";
+ }
+
+ @Override
+ public Reader readerValue() {
+ return null;
+ }
+
+ @Override
+ public Number numericValue() {
+ return null;
+ }
+
+ @Override
+ public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+ return null;
+ }
+ });
+ iw.addDocument(list);
+ fail("didn't get any exception, boost silently discarded");
+ } catch (UnsupportedOperationException expected) {
+ // expected
+ }
+ DirectoryReader ir = DirectoryReader.open(iw, false);
+ assertEquals(1, ir.numDocs());
+ assertEquals("sometext", ir.document(0).get("field1"));
+ ir.close();
+ iw.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Sat Aug 4 21:26:10 2012
@@ -23,6 +23,9 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
import org.apache.lucene.document.*;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.*;
import org.apache.lucene.util.*;
import org.junit.Test;
@@ -48,10 +51,13 @@ public class TestRollingUpdates extends
final int SIZE = atLeast(20);
int id = 0;
IndexReader r = null;
+ IndexSearcher s = null;
final int numUpdates = (int) (SIZE * (2+(TEST_NIGHTLY ? 200*random().nextDouble() : 5*random().nextDouble())));
if (VERBOSE) {
System.out.println("TEST: numUpdates=" + numUpdates);
}
+ int updateCount = 0;
+ // TODO: sometimes update ids not in order...
for(int docIter=0;docIter<numUpdates;docIter++) {
final Document doc = docs.nextDoc();
final String myID = ""+id;
@@ -60,16 +66,59 @@ public class TestRollingUpdates extends
} else {
id++;
}
+ if (VERBOSE) {
+ System.out.println(" docIter=" + docIter + " id=" + id);
+ }
((Field) doc.getField("docid")).setStringValue(myID);
- w.updateDocument(new Term("docid", myID), doc);
+
+ Term idTerm = new Term("docid", myID);
+
+ final boolean doUpdate;
+ if (s != null && updateCount < SIZE) {
+ TopDocs hits = s.search(new TermQuery(idTerm), 1);
+ assertEquals(1, hits.totalHits);
+ doUpdate = !w.tryDeleteDocument(r, hits.scoreDocs[0].doc);
+ if (VERBOSE) {
+ if (doUpdate) {
+ System.out.println(" tryDeleteDocument failed");
+ } else {
+ System.out.println(" tryDeleteDocument succeeded");
+ }
+ }
+ } else {
+ doUpdate = true;
+ if (VERBOSE) {
+ System.out.println(" no searcher: doUpdate=true");
+ }
+ }
+
+ updateCount++;
+
+ if (doUpdate) {
+ w.updateDocument(idTerm, doc);
+ } else {
+ w.addDocument(doc);
+ }
if (docIter >= SIZE && random().nextInt(50) == 17) {
if (r != null) {
r.close();
}
+
final boolean applyDeletions = random().nextBoolean();
+
+ if (VERBOSE) {
+ System.out.println("TEST: reopen applyDeletions=" + applyDeletions);
+ }
+
r = w.getReader(applyDeletions);
+ if (applyDeletions) {
+ s = new IndexSearcher(r);
+ } else {
+ s = null;
+ }
assertTrue("applyDeletions=" + applyDeletions + " r.numDocs()=" + r.numDocs() + " vs SIZE=" + SIZE, !applyDeletions || r.numDocs() == SIZE);
+ updateCount = 0;
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java Sat Aug 4 21:26:10 2012
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -189,6 +190,41 @@ public class TestFuzzyQuery extends Luce
directory.close();
}
+ public void test2() throws Exception {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
+ addDoc("LANGE", writer);
+ addDoc("LUETH", writer);
+ addDoc("PIRSING", writer);
+ addDoc("RIEGEL", writer);
+ addDoc("TRZECZIAK", writer);
+ addDoc("WALKER", writer);
+ addDoc("WBR", writer);
+ addDoc("WE", writer);
+ addDoc("WEB", writer);
+ addDoc("WEBE", writer);
+ addDoc("WEBER", writer);
+ addDoc("WEBERE", writer);
+ addDoc("WEBREE", writer);
+ addDoc("WEBEREI", writer);
+ addDoc("WBRE", writer);
+ addDoc("WITTKOPF", writer);
+ addDoc("WOJNAROWSKI", writer);
+ addDoc("WRICKE", writer);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ FuzzyQuery query = new FuzzyQuery(new Term("field", "WEBER"), 2, 1);
+ //query.setRewriteMethod(FuzzyQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(8, hits.length);
+
+ reader.close();
+ directory.close();
+ }
+
/**
* MultiTermQuery provides (via attribute) information about which values
* must be competitive to enter the priority queue.
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Sat Aug 4 21:26:10 2012
@@ -206,7 +206,7 @@ public class TestPositionIncrement exten
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer());
Document doc = new Document();
doc.add(new TextField("content", new StringReader(
- "a a b c d e a f g h i j a b k k"), Field.Store.NO));
+ "a a b c d e a f g h i j a b k k")));
writer.addDocument(doc);
final IndexReader readerFromWriter = writer.getReader();
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java Sat Aug 4 21:26:10 2012
@@ -77,8 +77,7 @@ public class TestShardSearching extends
System.out.println("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds);
}
- start(_TestUtil.getTempDir("TestShardSearching").toString(),
- numNodes,
+ start(numNodes,
runTimeSec,
maxSearcherAgeSeconds
);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/TestSort.java Sat Aug 4 21:26:10 2012
@@ -196,7 +196,7 @@ public class TestSort extends LuceneTest
if (data[i][11] != null) doc.add (new StringField ("parser", data[i][11], Field.Store.NO));
for(IndexableField f : doc.getFields()) {
- if (!f.fieldType().omitNorms()) {
+ if (f.fieldType().indexed() && !f.fieldType().omitNorms()) {
((Field) f).setBoost(2.0f);
}
}
@@ -239,7 +239,7 @@ public class TestSort extends LuceneTest
doc.add(new SortedBytesDocValuesField("string2", new BytesRef(num2)));
doc.add (new Field ("tracer2", num2, onlyStored));
for(IndexableField f2 : doc.getFields()) {
- if (!f2.fieldType().omitNorms()) {
+ if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
((Field) f2).setBoost(2.0f);
}
}
@@ -255,7 +255,7 @@ public class TestSort extends LuceneTest
doc.add (new Field ("tracer2_fixed", num2Fixed, onlyStored));
for(IndexableField f2 : doc.getFields()) {
- if (!f2.fieldType().omitNorms()) {
+ if (f2.fieldType().indexed() && !f2.fieldType().omitNorms()) {
((Field) f2).setBoost(2.0f);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Sat Aug 4 21:26:10 2012
@@ -256,7 +256,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
@@ -293,7 +293,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexSearcher is = newSearcher(reader);
@@ -328,7 +328,7 @@ public class TestPayloadSpans extends Lu
newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer()));
Document doc = new Document();
- doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"), Field.Store.NO));
+ doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexSearcher is = newSearcher(reader);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Sat Aug 4 21:26:10 2012
@@ -1096,13 +1096,6 @@ public class TestFSTs extends LuceneTest
// file, up until a time limit
public void testRealTerms() throws Exception {
- // TODO: is this necessary? we use the annotation...
- final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra");
- if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) {
- // no
- Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()));
- }
-
final LineFileDocs docs = new LineFileDocs(random(), true);
final int RUN_TIME_MSEC = atLeast(500);
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64);
Modified: lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java Sat Aug 4 21:26:10 2012
@@ -190,7 +190,7 @@ public class IndexFiles {
// so that the text of the file is tokenized and indexed, but not stored.
// Note that FileReader expects the file to be in UTF-8 encoding.
// If that's not the case searching for special characters will fail.
- doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")), Field.Store.NO));
+ doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8"))));
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
// New index, so we just add the document (no old document can be there):
Modified: lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Sat Aug 4 21:26:10 2012
@@ -440,21 +440,25 @@ public class TestSlowFuzzyQuery extends
assertEquals(1, hits.length);
assertEquals("foobar", searcher.doc(hits[0].doc).get("field"));
- q = new SlowFuzzyQuery(new Term("field", "t"), 3);
- hits = searcher.search(q, 10).scoreDocs;
- assertEquals(1, hits.length);
- assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ // TODO: cannot really be supported given the legacy scoring
+ // system which scores negative, if the distance > min term len,
+ // so such matches were always impossible with lucene 3.x, etc
+ //
+ //q = new SlowFuzzyQuery(new Term("field", "t"), 3);
+ //hits = searcher.search(q, 10).scoreDocs;
+ //assertEquals(1, hits.length);
+ //assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
- hits = searcher.search(q, 10).scoreDocs;
- assertEquals(1, hits.length);
- assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
+ // hits = searcher.search(q, 10).scoreDocs;
+ // assertEquals(1, hits.length);
+ // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
- hits = searcher.search(q, 10).scoreDocs;
- assertEquals(2, hits.length);
- assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
+ // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
+ // hits = searcher.search(q, 10).scoreDocs;
+ // assertEquals(2, hits.length);
+ // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ // assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
reader.close();
index.close();
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Sat Aug 4 21:26:10 2012
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
+import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings;
import org.apache.lucene.codecs.lucene40.Lucene40Codec;
import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
import org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds;
@@ -98,6 +99,10 @@ public class RandomCodec extends Lucene4
new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
// add pulsing again with (usually) different parameters
new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+ //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene40Postings to be constructed
+ //with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing
+ //with such "wrapper" classes?
+ new TestBloomFilteredLucene40Postings(),
new MockSepPostingsFormat(),
new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)),
new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)),
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java Sat Aug 4 21:26:10 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
*/
import java.io.Closeable;
-import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -27,15 +26,17 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.PrintStreamInfoStream;
+import org.apache.lucene.util._TestUtil;
// TODO
// - doc blocks? so we can test joins/grouping...
@@ -423,11 +424,16 @@ public abstract class ShardSearchingTest
private volatile ShardIndexSearcher currentShardSearcher;
- public NodeState(Random random, String baseDir, int nodeID, int numNodes) throws IOException {
+ public NodeState(Random random, int nodeID, int numNodes) throws IOException {
myNodeID = nodeID;
- dir = newFSDirectory(new File(baseDir + "." + myNodeID));
+ dir = newFSDirectory(_TestUtil.getTempDir("ShardSearchingTestBase"));
// TODO: set warmer
- writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+ if (VERBOSE) {
+ iwc.setInfoStream(new PrintStreamInfoStream(System.out));
+ }
+ writer = new IndexWriter(dir, iwc);
mgr = new SearcherManager(writer, true, null);
searchers = new SearcherLifetimeManager();
@@ -556,14 +562,14 @@ public abstract class ShardSearchingTest
long endTimeNanos;
private Thread changeIndicesThread;
- protected void start(String baseDirName, int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException {
+ protected void start(int numNodes, double runTimeSec, int maxSearcherAgeSeconds) throws IOException {
endTimeNanos = System.nanoTime() + (long) (runTimeSec*1000000000);
this.maxSearcherAgeSeconds = maxSearcherAgeSeconds;
nodes = new NodeState[numNodes];
for(int nodeID=0;nodeID<numNodes;nodeID++) {
- nodes[nodeID] = new NodeState(random(), baseDirName, nodeID, numNodes);
+ nodes[nodeID] = new NodeState(random(), nodeID, numNodes);
}
long[] nodeVersions = new long[nodes.length];
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java Sat Aug 4 21:26:10 2012
@@ -31,8 +31,6 @@ import org.apache.lucene.util._TestUtil;
public class BaseDirectoryWrapper extends Directory {
/** our in directory */
protected final Directory delegate;
- /** best effort: base on in Directory is volatile */
- protected boolean open;
private boolean checkIndexOnClose = true;
private boolean crossCheckTermVectorsOnClose = true;
@@ -43,7 +41,7 @@ public class BaseDirectoryWrapper extend
@Override
public void close() throws IOException {
- open = false;
+ isOpen = false;
if (checkIndexOnClose && indexPossiblyExists()) {
_TestUtil.checkIndex(this, crossCheckTermVectorsOnClose);
}
@@ -51,7 +49,7 @@ public class BaseDirectoryWrapper extend
}
public boolean isOpen() {
- return open;
+ return isOpen;
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Sat Aug 4 21:26:10 2012
@@ -551,7 +551,7 @@ public class MockDirectoryWrapper extend
if (noDeleteOpenFile && openLocks.size() > 0) {
throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
}
- open = false;
+ isOpen = false;
if (getCheckIndexOnClose()) {
if (indexPossiblyExists()) {
if (LuceneTestCase.VERBOSE) {
@@ -614,11 +614,6 @@ public class MockDirectoryWrapper extend
public synchronized void removeIndexInput(IndexInput in, String name) {
removeOpenFile(in, name);
}
-
- @Override
- public synchronized boolean isOpen() {
- return open;
- }
/**
* Objects that represent fail-able conditions. Objects of a derived
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1369470&r1=1369469&r2=1369470&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug 4 21:26:10 2012
@@ -20,5 +20,6 @@ org.apache.lucene.codecs.mocksep.MockSep
org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat
org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat
org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds
+org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings
org.apache.lucene.codecs.asserting.AssertingPostingsFormat