You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/12 10:15:38 UTC
svn commit: r1034304 [3/7] - in /lucene/dev/branches/docvalues: ./ lucene/
lucene/contrib/ lucene/contrib/benchmark/conf/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/contrib/benchmark/src/test/org/apache/lucene/be...
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Fri Nov 12 09:15:30 2010
@@ -28,13 +28,19 @@ import org.apache.lucene.index.DocsAndPo
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.UnicodeUtil;
import java.io.IOException;
import java.util.Comparator;
import java.util.Map;
+import java.util.Set;
import java.util.HashMap;
+import java.util.TreeMap;
+import java.util.SortedMap;
+import java.util.Iterator;
class SimpleTextFieldsReader extends FieldsProducer {
@@ -79,7 +85,7 @@ class SimpleTextFieldsReader extends Fie
private class SimpleTextFieldsEnum extends FieldsEnum {
private final IndexInput in;
private final BytesRef scratch = new BytesRef(10);
- private boolean omitTF;
+ private String current;
public SimpleTextFieldsEnum() {
this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
@@ -90,11 +96,12 @@ class SimpleTextFieldsReader extends Fie
while(true) {
readLine(in, scratch);
if (scratch.equals(END)) {
+ current = null;
return null;
}
if (scratch.startsWith(FIELD)) {
String field = StringHelper.intern(new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, "UTF-8"));
- omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+ current = field;
return field;
}
}
@@ -102,7 +109,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public TermsEnum terms() throws IOException {
- return new SimpleTextTermsEnum(in.getFilePointer(), omitTF);
+ return SimpleTextFieldsReader.this.terms(current).iterator();
}
@Override
@@ -116,21 +123,42 @@ class SimpleTextFieldsReader extends Fie
private final IndexInput in;
private final boolean omitTF;
private BytesRef current;
- private final long fieldStart;
- private final BytesRef scratch = new BytesRef(10);
- private final BytesRef scratch2 = new BytesRef(10);
private int docFreq;
private long docsStart;
private boolean ended;
+ private final TreeMap<BytesRef,TermData> allTerms;
+ private Iterator<Map.Entry<BytesRef,TermData>> iter;
- public SimpleTextTermsEnum(long offset, boolean omitTF) throws IOException {
+ public SimpleTextTermsEnum(TreeMap<BytesRef,TermData> allTerms, boolean omitTF) throws IOException {
this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
- this.in.seek(offset);
+ this.allTerms = allTerms;
this.omitTF = omitTF;
- fieldStart = offset;
+ iter = allTerms.entrySet().iterator();
}
public SeekStatus seek(BytesRef text, boolean useCache /* ignored */) throws IOException {
+
+ final SortedMap<BytesRef,TermData> tailMap = allTerms.tailMap(text);
+
+ if (tailMap.isEmpty()) {
+ current = null;
+ return SeekStatus.END;
+ } else {
+ current = tailMap.firstKey();
+ final TermData td = tailMap.get(current);
+ docsStart = td.docsStart;
+ docFreq = td.docFreq;
+ iter = tailMap.entrySet().iterator();
+ assert iter.hasNext();
+ iter.next();
+ if (current.equals(text)) {
+ return SeekStatus.FOUND;
+ } else {
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+ /*
if (current != null) {
final int cmp = current.compareTo(text);
if (cmp == 0) {
@@ -160,6 +188,7 @@ class SimpleTextFieldsReader extends Fie
current = null;
ended = true;
return SeekStatus.END;
+ */
}
@Override
@@ -169,6 +198,20 @@ class SimpleTextFieldsReader extends Fie
@Override
public BytesRef next() throws IOException {
assert !ended;
+
+ if (iter.hasNext()) {
+ Map.Entry<BytesRef,TermData> ent = iter.next();
+ current = ent.getKey();
+ TermData td = ent.getValue();
+ docFreq = td.docFreq;
+ docsStart = td.docsStart;
+ return current;
+ } else {
+ current = null;
+ return null;
+ }
+
+ /*
readLine(in, scratch);
if (scratch.equals(END) || scratch.startsWith(FIELD)) {
ended = true;
@@ -199,6 +242,7 @@ class SimpleTextFieldsReader extends Fie
in.seek(lineStart);
return current;
}
+ */
}
@Override
@@ -261,7 +305,8 @@ class SimpleTextFieldsReader extends Fie
private int tf;
private Bits skipDocs;
private final BytesRef scratch = new BytesRef(10);
-
+ private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
+
public SimpleTextDocsEnum() {
this.inStart = SimpleTextFieldsReader.this.in;
this.in = (IndexInput) this.inStart.clone();
@@ -309,7 +354,8 @@ class SimpleTextFieldsReader extends Fie
}
return docID;
}
- docID = Integer.parseInt(new String(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length));
+ UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
+ docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
termFreq = 0;
first = false;
} else if (scratch.startsWith(POS)) {
@@ -346,6 +392,8 @@ class SimpleTextFieldsReader extends Fie
private Bits skipDocs;
private final BytesRef scratch = new BytesRef(10);
private final BytesRef scratch2 = new BytesRef(10);
+ private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
+ private final UnicodeUtil.UTF16Result scratchUTF16_2 = new UnicodeUtil.UTF16Result();
private BytesRef payload;
private long nextDocStart;
@@ -388,7 +436,8 @@ class SimpleTextFieldsReader extends Fie
in.seek(posStart);
return docID;
}
- docID = Integer.parseInt(new String(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length));
+ UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
+ docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
tf = 0;
posStart = in.getFilePointer();
first = false;
@@ -419,7 +468,8 @@ class SimpleTextFieldsReader extends Fie
public int nextPosition() throws IOException {
readLine(in, scratch);
assert scratch.startsWith(POS): "got line=" + scratch.utf8ToString();
- final int pos = Integer.parseInt(new String(scratch.bytes, scratch.offset+POS.length, scratch.length-POS.length));
+ UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+POS.length, scratch.length-POS.length, scratchUTF16_2);
+ final int pos = ArrayUtil.parseInt(scratchUTF16_2.result, 0, scratchUTF16_2.length);
final long fp = in.getFilePointer();
readLine(in, scratch);
if (scratch.startsWith(PAYLOAD)) {
@@ -454,20 +504,70 @@ class SimpleTextFieldsReader extends Fie
}
}
+ static class TermData {
+ public long docsStart;
+ public int docFreq;
+
+ public TermData(long docsStart, int docFreq) {
+ this.docsStart = docsStart;
+ this.docFreq = docFreq;
+ }
+ }
+
private class SimpleTextTerms extends Terms {
private final String field;
private final long termsStart;
private final boolean omitTF;
- public SimpleTextTerms(String field, long termsStart) {
+ // NOTE: horribly, horribly RAM consuming, but then
+ // SimpleText should never be used in production
+ private final TreeMap<BytesRef,TermData> allTerms = new TreeMap<BytesRef,TermData>();
+
+ private final BytesRef scratch = new BytesRef(10);
+
+ public SimpleTextTerms(String field, long termsStart) throws IOException {
this.field = StringHelper.intern(field);
this.termsStart = termsStart;
omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+ loadTerms();
+ }
+
+ private void loadTerms() throws IOException {
+ IndexInput in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
+ in.seek(termsStart);
+ final BytesRef lastTerm = new BytesRef(10);
+ long lastDocsStart = -1;
+ int docFreq = 0;
+ while(true) {
+ readLine(in, scratch);
+ if (scratch.equals(END) || scratch.startsWith(FIELD)) {
+ if (lastDocsStart != -1) {
+ allTerms.put(new BytesRef(lastTerm),
+ new TermData(lastDocsStart, docFreq));
+ }
+ break;
+ } else if (scratch.startsWith(DOC)) {
+ docFreq++;
+ } else if (scratch.startsWith(TERM)) {
+ if (lastDocsStart != -1) {
+ allTerms.put(new BytesRef(lastTerm),
+ new TermData(lastDocsStart, docFreq));
+ }
+ lastDocsStart = in.getFilePointer();
+ final int len = scratch.length - TERM.length;
+ if (len > lastTerm.length) {
+ lastTerm.grow(len);
+ }
+ System.arraycopy(scratch.bytes, TERM.length, lastTerm.bytes, 0, len);
+ lastTerm.length = len;
+ docFreq = 0;
+ }
+ }
}
@Override
public TermsEnum iterator() throws IOException {
- return new SimpleTextTermsEnum(termsStart, omitTF);
+ return new SimpleTextTermsEnum(allTerms, omitTF);
}
@Override
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java Fri Nov 12 09:15:30 2010
@@ -1045,7 +1045,7 @@ public abstract class QueryParserBase {
String termImage=discardEscapeChar(term.image);
if (wildcard) {
- q = getWildcardQuery(qfield, termImage);
+ q = getWildcardQuery(qfield, term.image);
} else if (prefix) {
q = getPrefixQuery(qfield,
discardEscapeChar(term.image.substring
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Fri Nov 12 09:15:30 2010
@@ -55,7 +55,7 @@ public final class FuzzyTermsEnum extend
attributes().addAttribute(MultiTermQuery.BoostAttribute.class);
private final MultiTermQuery.MaxNonCompetitiveBoostAttribute maxBoostAtt;
- private final Priv.LevenshteinAutomataAttribute dfaAtt;
+ private final LevenshteinAutomataAttribute dfaAtt;
private float bottom;
private BytesRef bottomTerm;
@@ -110,7 +110,7 @@ public final class FuzzyTermsEnum extend
for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp))
termText[j++] = cp = utf16.codePointAt(i);
this.termLength = termText.length;
- this.dfaAtt = atts.addAttribute(Priv.LevenshteinAutomataAttribute.class);
+ this.dfaAtt = atts.addAttribute(LevenshteinAutomataAttribute.class);
//The prefix could be longer than the word.
//It's kind of silly though. It means we must match the entire word.
@@ -552,49 +552,44 @@ public final class FuzzyTermsEnum extend
return scale_factor;
}
- // Wrapper class to hide the attribute from outside!
- private static final class Priv {
-
- /** @lucene.internal */
- public static interface LevenshteinAutomataAttribute extends Attribute {
- public List<ByteRunAutomaton> automata();
- }
+ /** @lucene.internal */
+ public static interface LevenshteinAutomataAttribute extends Attribute {
+ public List<ByteRunAutomaton> automata();
+ }
- /** @lucene.internal */
- public static final class LevenshteinAutomataAttributeImpl extends AttributeImpl implements LevenshteinAutomataAttribute {
- private final List<ByteRunAutomaton> automata = new ArrayList<ByteRunAutomaton>();
+ /** @lucene.internal */
+ public static final class LevenshteinAutomataAttributeImpl extends AttributeImpl implements LevenshteinAutomataAttribute {
+ private final List<ByteRunAutomaton> automata = new ArrayList<ByteRunAutomaton>();
- public List<ByteRunAutomaton> automata() {
- return automata;
- }
+ public List<ByteRunAutomaton> automata() {
+ return automata;
+ }
- @Override
- public void clear() {
- automata.clear();
- }
+ @Override
+ public void clear() {
+ automata.clear();
+ }
- @Override
- public int hashCode() {
- return automata.hashCode();
- }
+ @Override
+ public int hashCode() {
+ return automata.hashCode();
+ }
- @Override
- public boolean equals(Object other) {
- if (this == other)
- return true;
- if (!(other instanceof LevenshteinAutomataAttributeImpl))
- return false;
- return automata.equals(((LevenshteinAutomataAttributeImpl) other).automata);
- }
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof LevenshteinAutomataAttributeImpl))
+ return false;
+ return automata.equals(((LevenshteinAutomataAttributeImpl) other).automata);
+ }
- @Override
- public void copyTo(AttributeImpl target) {
- final List<ByteRunAutomaton> targetAutomata =
- ((LevenshteinAutomataAttribute) target).automata();
- targetAutomata.clear();
- targetAutomata.addAll(automata);
- }
+ @Override
+ public void copyTo(AttributeImpl target) {
+ final List<ByteRunAutomaton> targetAutomata =
+ ((LevenshteinAutomataAttribute) target).automata();
+ targetAutomata.clear();
+ targetAutomata.addAll(automata);
}
-
}
}
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:967125-979432
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:1021634-1034284
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:824912-931101
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334,948516
/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924483-924731,924781,925176-925462
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java Fri Nov 12 09:15:30 2010
@@ -45,6 +45,9 @@ public class WildcardQuery extends Autom
/** Char equality with support for wildcards */
public static final char WILDCARD_CHAR = '?';
+ /** Escape character */
+ public static final char WILDCARD_ESCAPE = '\\';
+
/**
* Constructs a query for terms matching <code>term</code>.
*/
@@ -56,6 +59,7 @@ public class WildcardQuery extends Autom
* Convert Lucene wildcard syntax into an automaton.
* @lucene.internal
*/
+ @SuppressWarnings("fallthrough")
public static Automaton toAutomaton(Term wildcardquery) {
List<Automaton> automata = new ArrayList<Automaton>();
@@ -63,6 +67,7 @@ public class WildcardQuery extends Autom
for (int i = 0; i < wildcardText.length();) {
final int c = wildcardText.codePointAt(i);
+ int length = Character.charCount(c);
switch(c) {
case WILDCARD_STRING:
automata.add(BasicAutomata.makeAnyString());
@@ -70,10 +75,18 @@ public class WildcardQuery extends Autom
case WILDCARD_CHAR:
automata.add(BasicAutomata.makeAnyChar());
break;
+ case WILDCARD_ESCAPE:
+ // add the next codepoint instead, if it exists
+ if (i + length < wildcardText.length()) {
+ final int nextChar = wildcardText.codePointAt(i + length);
+ length += Character.charCount(nextChar);
+ automata.add(BasicAutomata.makeChar(nextChar));
+ break;
+ } // else fallthru, lenient parsing with a trailing \
default:
automata.add(BasicAutomata.makeChar(c));
}
- i += Character.charCount(c);
+ i += length;
}
return BasicOperations.concatenate(automata);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java Fri Nov 12 09:15:30 2010
@@ -197,6 +197,11 @@ public abstract class Directory implemen
return this.toString();
}
+ @Override
+ public String toString() {
+ return super.toString() + " lockFactory=" + getLockFactory();
+ }
+
/**
* Copies the file <i>src</i> to {@link Directory} <i>to</i> under the new
* file name <i>dest</i>.
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Nov 12 09:15:30 2010
@@ -398,7 +398,7 @@ public abstract class FSDirectory extend
/** For debug output. */
@Override
public String toString() {
- return this.getClass().getName() + "@" + directory;
+ return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
}
/**
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.util; // from
/** A variety of high efficiency bit twiddling routines.
* @lucene.internal
*/
-public class BitUtil {
+public final class BitUtil {
+
+ private BitUtil() {} // no instance
/** Returns the number of bits set in the long */
public static int pop(long x) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java Fri Nov 12 09:15:30 2010
@@ -31,6 +31,8 @@ import java.io.IOException;
*/
public final class CodecUtil {
+ private CodecUtil() {} // no instance
+
private final static int CODEC_MAGIC = 0x3fd76c17;
public static IndexOutput writeHeader(IndexOutput out, String codec, int version)
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.util;
/**
* @lucene.internal
*/
-public class English {
+public final class English {
+
+ private English() {} // no instance
public static String longToEnglish(long i) {
StringBuilder result = new StringBuilder();
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java Fri Nov 12 09:15:30 2010
@@ -23,7 +23,10 @@ import java.io.IOException;
import org.apache.lucene.store.DataOutput;
/** @lucene.internal */
-public class IOUtils {
+public final class IOUtils {
+
+ private IOUtils() {} // no instance
+
/**
* <p>Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. Some of the <tt>Closeable</tt>s
* may be null, they are ignored. After everything is closed, method either throws <tt>priorException</tt>,
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java Fri Nov 12 09:15:30 2010
@@ -53,7 +53,7 @@ import java.nio.ByteBuffer;
*
* @lucene.experimental
*/
-public class IndexableBinaryStringTools {
+public final class IndexableBinaryStringTools {
private static final CodingCase[] CODING_CASES = {
// CodingCase(int initialShift, int finalShift)
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Fri Nov 12 09:15:30 2010
@@ -28,7 +28,9 @@ import org.apache.lucene.index.IndexRead
*
* @lucene.internal
*/
-public class ReaderUtil {
+public final class ReaderUtil {
+
+ private ReaderUtil() {} // no instance
public static class Slice {
public static final Slice[] EMPTY_ARRAY = new Slice[0];
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,10 @@ package org.apache.lucene.util;
/**
* Helper methods to ease implementing {@link Object#toString()}.
*/
-public class ToStringUtils {
+public final class ToStringUtils {
+
+ private ToStringUtils() {} // no instance
+
/**
* for printing boost only if not 1.0
*/
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java Fri Nov 12 09:15:30 2010
@@ -93,7 +93,9 @@ package org.apache.lucene.util;
* @lucene.internal
*/
-final public class UnicodeUtil {
+public final class UnicodeUtil {
+
+ private UnicodeUtil() {} // no instance
public static final int UNI_SUR_HIGH_START = 0xD800;
public static final int UNI_SUR_HIGH_END = 0xDBFF;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java Fri Nov 12 09:15:30 2010
@@ -178,7 +178,7 @@ final public class SpecialOperations {
* Reverses the language of the given (non-singleton) automaton while returning
* the set of new initial states.
*/
- static Set<State> reverse(Automaton a) {
+ public static Set<State> reverse(Automaton a) {
a.expandSingleton();
// reverse all edges
HashMap<State, HashSet<Transition>> m = new HashMap<State, HashSet<Transition>>();
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Fri Nov 12 09:15:30 2010
@@ -73,7 +73,10 @@ public class TestExternalCodecs extends
// - good improvement would be to write through to disk,
// and then load into ram from disk
public static class RAMOnlyCodec extends Codec {
-
+
+ public RAMOnlyCodec() {
+ name = "RamOnly";
+ }
// Postings state:
static class RAMPostings extends FieldsProducer {
final Map<String,RAMField> fieldToTerms = new TreeMap<String,RAMField>();
@@ -482,6 +485,7 @@ public class TestExternalCodecs extends
@Override
public FieldsProducer fieldsProducer(SegmentReadState readState)
throws IOException {
+
return state.get(readState.segmentInfo.name);
}
@@ -495,20 +499,10 @@ public class TestExternalCodecs extends
}
public static class MyCodecs extends CodecProvider {
- PerFieldCodecWrapper perField;
-
MyCodecs() {
Codec ram = new RAMOnlyCodec();
- Codec pulsing = new PulsingReverseTermsCodec();
- perField = new PerFieldCodecWrapper(ram);
- perField.add("field2", pulsing);
- perField.add("id", pulsing);
- register(perField);
- }
-
- @Override
- public Codec getWriter(SegmentWriteState state) {
- return perField;
+ register(ram);
+ setDefaultFieldCodec(ram.name);
}
}
@@ -631,20 +625,28 @@ public class TestExternalCodecs extends
// whose term sort is backwards unicode code point, and
// storing "field1" as a custom entirely-in-RAM codec
public void testPerFieldCodec() throws Exception {
+ CodecProvider provider = new MyCodecs();
+ Codec pulsing = new PulsingReverseTermsCodec();
+ provider.register(pulsing);
+
final int NUM_DOCS = 173;
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(new MyCodecs()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(provider));
w.setMergeFactor(3);
Document doc = new Document();
// uses default codec:
doc.add(newField("field1", "this field uses the standard codec as the test", Field.Store.NO, Field.Index.ANALYZED));
// uses pulsing codec:
- doc.add(newField("field2", "this field uses the pulsing codec as the test", Field.Store.NO, Field.Index.ANALYZED));
+ Field field2 = newField("field2", "this field uses the pulsing codec as the test", Field.Store.NO, Field.Index.ANALYZED);
+ provider.setFieldCodec(field2.name(), pulsing.name);
+ doc.add(field2);
Field idField = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
+ provider.setFieldCodec(idField.name(), pulsing.name);
+
doc.add(idField);
for(int i=0;i<NUM_DOCS;i++) {
idField.setValue(""+i);
Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestDateTools.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:1021634-1034284
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:1021634-1034284
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334,948516
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java Fri Nov 12 09:15:30 2010
@@ -42,6 +42,7 @@ public class RandomIndexWriter implement
private final Random r;
int docCount;
int flushAt;
+ private boolean getReaderCalled;
// Randomly calls Thread.yield so we mixup thread scheduling
private static final class MockIndexWriter extends IndexWriter {
@@ -123,10 +124,13 @@ public class RandomIndexWriter implement
}
public IndexReader getReader() throws IOException {
+ getReaderCalled = true;
+ if (r.nextInt(4) == 2)
+ w.optimize();
// If we are writing with PreFlexRW, force a full
// IndexReader.open so terms are sorted in codepoint
// order during searching:
- if (!w.codecs.getWriter(null).name.equals("PreFlex") && r.nextBoolean()) {
+ if (!w.codecs.getDefaultFieldCodec().equals("PreFlex") && r.nextBoolean()) {
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW.getReader: use NRT reader");
}
@@ -141,7 +145,9 @@ public class RandomIndexWriter implement
}
public void close() throws IOException {
- if (r.nextInt(4) == 2) {
+ // if someone isn't using getReader() API, we want to be sure to
+ // maybeOptimize since presumably they might open a reader on the dir.
+ if (getReaderCalled == false && r.nextInt(4) == 2) {
w.optimize();
}
w.close();
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Nov 12 09:15:30 2010
@@ -18,20 +18,26 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
+import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.PhraseQuery;
-
public class TestAddIndexes extends LuceneTestCase {
public void testSimpleCase() throws IOException {
@@ -563,4 +569,365 @@ public class TestAddIndexes extends Luce
dir2.close();
}
+ // TODO: these are also in TestIndexWriter... add a simple doc-writing method
+ // like this to LuceneTestCase?
+ private void addDoc(IndexWriter writer) throws IOException
+ {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+
+ private abstract class RunAddIndexesThreads {
+
+ Directory dir, dir2;
+ final static int NUM_INIT_DOCS = 17;
+ IndexWriter writer2;
+ final List<Throwable> failures = new ArrayList<Throwable>();
+ volatile boolean didClose;
+ final IndexReader[] readers;
+ final int NUM_COPY;
+ final static int NUM_THREADS = 5;
+ final Thread[] threads = new Thread[NUM_THREADS];
+
+ public RunAddIndexesThreads(int numCopy) throws Throwable {
+ NUM_COPY = numCopy;
+ dir = new MockDirectoryWrapper(new RAMDirectory());
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer())
+ .setMaxBufferedDocs(2));
+ for (int i = 0; i < NUM_INIT_DOCS; i++)
+ addDoc(writer);
+ writer.close();
+
+ dir2 = new MockDirectoryWrapper(new RAMDirectory());
+ writer2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer2.commit();
+
+ readers = new IndexReader[NUM_COPY];
+ for(int i=0;i<NUM_COPY;i++)
+ readers[i] = IndexReader.open(dir, true);
+ }
+
+ void launchThreads(final int numIter) {
+
+ for(int i=0;i<NUM_THREADS;i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+
+ final Directory[] dirs = new Directory[NUM_COPY];
+ for(int k=0;k<NUM_COPY;k++)
+ dirs[k] = new MockDirectoryWrapper(new RAMDirectory(dir));
+
+ int j=0;
+
+ while(true) {
+ // System.out.println(Thread.currentThread().getName() + ": iter j=" + j);
+ if (numIter > 0 && j == numIter)
+ break;
+ doBody(j++, dirs);
+ }
+ } catch (Throwable t) {
+ handle(t);
+ }
+ }
+ };
+ }
+
+ for(int i=0;i<NUM_THREADS;i++)
+ threads[i].start();
+ }
+
+ void joinThreads() throws Exception {
+ for(int i=0;i<NUM_THREADS;i++)
+ threads[i].join();
+ }
+
+ void close(boolean doWait) throws Throwable {
+ didClose = true;
+ writer2.close(doWait);
+ }
+
+ void closeDir() throws Throwable {
+ for(int i=0;i<NUM_COPY;i++)
+ readers[i].close();
+ dir2.close();
+ }
+
+ abstract void doBody(int j, Directory[] dirs) throws Throwable;
+ abstract void handle(Throwable t);
+ }
+
+ private class CommitAndAddIndexes extends RunAddIndexesThreads {
+ public CommitAndAddIndexes(int numCopy) throws Throwable {
+ super(numCopy);
+ }
+
+ @Override
+ void handle(Throwable t) {
+ t.printStackTrace(System.out);
+ synchronized(failures) {
+ failures.add(t);
+ }
+ }
+
+ @Override
+ void doBody(int j, Directory[] dirs) throws Throwable {
+ switch(j%5) {
+ case 0:
+ writer2.addIndexes(dirs);
+ writer2.optimize();
+ break;
+ case 1:
+ writer2.addIndexes(dirs);
+ break;
+ case 2:
+ writer2.addIndexes(readers);
+ break;
+ case 3:
+ writer2.addIndexes(dirs);
+ writer2.maybeMerge();
+ break;
+ case 4:
+ writer2.commit();
+ }
+ }
+ }
+
+ // LUCENE-1335: test simultaneous addIndexes & commits
+ // from multiple threads
+ public void testAddIndexesWithThreads() throws Throwable {
+
+ final int NUM_ITER = 15;
+ final int NUM_COPY = 3;
+ CommitAndAddIndexes c = new CommitAndAddIndexes(NUM_COPY);
+ c.launchThreads(NUM_ITER);
+
+ for(int i=0;i<100;i++)
+ addDoc(c.writer2);
+
+ c.joinThreads();
+
+ int expectedNumDocs = 100+NUM_COPY*(4*NUM_ITER/5)*RunAddIndexesThreads.NUM_THREADS*RunAddIndexesThreads.NUM_INIT_DOCS;
+ assertEquals(expectedNumDocs, c.writer2.numDocs());
+
+ c.close(true);
+
+ assertTrue(c.failures.size() == 0);
+
+ _TestUtil.checkIndex(c.dir2);
+
+ IndexReader reader = IndexReader.open(c.dir2, true);
+ assertEquals(expectedNumDocs, reader.numDocs());
+ reader.close();
+
+ c.closeDir();
+ }
+
+ private class CommitAndAddIndexes2 extends CommitAndAddIndexes {
+ public CommitAndAddIndexes2(int numCopy) throws Throwable {
+ super(numCopy);
+ }
+
+ @Override
+ void handle(Throwable t) {
+ if (!(t instanceof AlreadyClosedException) && !(t instanceof NullPointerException)) {
+ t.printStackTrace(System.out);
+ synchronized(failures) {
+ failures.add(t);
+ }
+ }
+ }
+ }
+
+ // LUCENE-1335: test simultaneous addIndexes & close
+ public void testAddIndexesWithClose() throws Throwable {
+ final int NUM_COPY = 3;
+ CommitAndAddIndexes2 c = new CommitAndAddIndexes2(NUM_COPY);
+ //c.writer2.setInfoStream(System.out);
+ c.launchThreads(-1);
+
+ // Close w/o first stopping/joining the threads
+ c.close(true);
+ //c.writer2.close();
+
+ c.joinThreads();
+
+ _TestUtil.checkIndex(c.dir2);
+
+ c.closeDir();
+
+ assertTrue(c.failures.size() == 0);
+ }
+
+ private class CommitAndAddIndexes3 extends RunAddIndexesThreads {
+ public CommitAndAddIndexes3(int numCopy) throws Throwable {
+ super(numCopy);
+ }
+
+ @Override
+ void doBody(int j, Directory[] dirs) throws Throwable {
+ switch(j%5) {
+ case 0:
+ writer2.addIndexes(dirs);
+ writer2.optimize();
+ break;
+ case 1:
+ writer2.addIndexes(dirs);
+ break;
+ case 2:
+ writer2.addIndexes(readers);
+ break;
+ case 3:
+ writer2.optimize();
+ break;
+ case 4:
+ writer2.commit();
+ }
+ }
+
+ @Override
+ void handle(Throwable t) {
+ boolean report = true;
+
+ if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) {
+ report = !didClose;
+ } else if (t instanceof IOException) {
+ Throwable t2 = t.getCause();
+ if (t2 instanceof MergePolicy.MergeAbortedException) {
+ report = !didClose;
+ }
+ }
+ if (report) {
+ t.printStackTrace(System.out);
+ synchronized(failures) {
+ failures.add(t);
+ }
+ }
+ }
+ }
+
+ // LUCENE-1335: test simultaneous addIndexes & close
+ public void testAddIndexesWithCloseNoWait() throws Throwable {
+
+ final int NUM_COPY = 50;
+ CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
+ c.launchThreads(-1);
+
+ Thread.sleep(500);
+
+ // Close w/o first stopping/joining the threads
+ c.close(false);
+
+ c.joinThreads();
+
+ _TestUtil.checkIndex(c.dir2);
+
+ c.closeDir();
+
+ assertTrue(c.failures.size() == 0);
+ }
+
+ // LUCENE-1335: test simultaneous addIndexes & close
+ public void testAddIndexesWithRollback() throws Throwable {
+
+ final int NUM_COPY = 50;
+ CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
+ c.launchThreads(-1);
+
+ Thread.sleep(500);
+
+ // Close w/o first stopping/joining the threads
+ c.didClose = true;
+ c.writer2.rollback();
+
+ c.joinThreads();
+
+ _TestUtil.checkIndex(c.dir2);
+
+ c.closeDir();
+
+ assertTrue(c.failures.size() == 0);
+ }
+
+ private void addDocs3(IndexWriter writer, int numDocs) throws IOException {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+ doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+ }
+
+ public void testSimpleCaseCustomCodecProvider() throws IOException {
+ // main directory
+ Directory dir = newDirectory();
+ // two auxiliary directories
+ Directory aux = newDirectory();
+ Directory aux2 = newDirectory();
+ CodecProvider provider = new MockCodecProvider();
+ IndexWriter writer = null;
+
+ writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+ provider));
+ // add 100 documents
+ addDocs3(writer, 100);
+ assertEquals(100, writer.maxDoc());
+ writer.commit();
+ writer.close();
+ _TestUtil.checkIndex(dir, provider);
+
+ writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+ provider));
+ ((LogMergePolicy) writer.getConfig().getMergePolicy())
+ .setUseCompoundFile(false); // use one without a compound file
+ ((LogMergePolicy) writer.getConfig().getMergePolicy())
+ .setUseCompoundDocStore(false); // use one without a compound file
+ // add 40 documents in separate files
+ addDocs(writer, 40);
+ assertEquals(40, writer.maxDoc());
+ writer.commit();
+ writer.close();
+
+ writer = newWriter(aux2, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+ provider));
+ // add 40 documents in compound files
+ addDocs2(writer, 50);
+ assertEquals(50, writer.maxDoc());
+ writer.commit();
+ writer.close();
+
+ // test doc count before segments are merged
+ writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setCodecProvider(
+ provider));
+ assertEquals(100, writer.maxDoc());
+ writer.addIndexes(new Directory[] { aux, aux2 });
+ assertEquals(190, writer.maxDoc());
+ writer.close();
+ _TestUtil.checkIndex(dir, provider);
+
+ dir.close();
+ aux.close();
+ aux2.close();
+ }
+
+ public static class MockCodecProvider extends CodecProvider {
+ public MockCodecProvider() {
+ StandardCodec standardCodec = new StandardCodec();
+ SimpleTextCodec simpleTextCodec = new SimpleTextCodec();
+ MockSepCodec mockSepCodec = new MockSepCodec();
+ register(standardCodec);
+ register(mockSepCodec);
+ register(simpleTextCodec);
+ setFieldCodec("id", simpleTextCodec.name);
+ setFieldCodec("content", mockSepCodec.name);
+ }
+ }
+
}
Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:1021634-1034284
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334,948516
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java Fri Nov 12 09:15:30 2010
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -271,10 +270,10 @@ public class TestCodecs extends LuceneTe
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null));
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()));
si.setHasProx(false);
- final FieldsProducer reader = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
+ final FieldsProducer reader = si.getCodecInfo().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
final FieldsEnum fieldsEnum = reader.iterator();
assertNotNull(fieldsEnum.next());
@@ -319,9 +318,9 @@ public class TestCodecs extends LuceneTe
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null));
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()));
- final FieldsProducer terms = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
+ final FieldsProducer terms = si.getCodecInfo().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
final Verify[] threads = new Verify[NUM_TEST_THREADS-1];
for(int i=0;i<NUM_TEST_THREADS-1;i++) {
@@ -402,13 +401,9 @@ public class TestCodecs extends LuceneTe
protected MockSepCodecs() {
this.register(new MockSepCodec());
+ this.setDefaultFieldCodec("MockSep");
}
-
- @Override
- public Codec getWriter(final SegmentWriteState state) {
- return this.lookup("MockSep");
- }
-
+
}
private class Verify extends Thread {
@@ -611,11 +606,10 @@ public class TestCodecs extends LuceneTe
private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
final int termIndexInterval = this.nextInt(13, 27);
+ final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
+ final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, codecInfo);
- final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval,
- CodecProvider.getDefault());
-
- final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+ final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
Arrays.sort(fields);
for (final FieldData field : fields) {
field.write(consumer);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java Fri Nov 12 09:15:30 2010
@@ -197,7 +197,7 @@ public class TestDoc extends LuceneTestC
merger.closeReaders();
final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
- useCompoundFile, -1, null, false, merger.hasProx(), merger.getCodec());
+ useCompoundFile, -1, null, false, merger.hasProx(), merger.getSegmentCodecs());
if (useCompoundFile) {
List<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Fri Nov 12 09:15:30 2010
@@ -1813,4 +1813,29 @@ public class TestIndexReader extends Luc
r.close();
dir.close();
}
+
+ // LUCENE-2753
+ public void testListCommits() throws Exception {
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, null).setIndexDeletionPolicy(sdp));
+ writer.addDocument(new Document());
+ writer.commit();
+ sdp.snapshot("c1");
+ writer.addDocument(new Document());
+ writer.commit();
+ sdp.snapshot("c2");
+ writer.addDocument(new Document());
+ writer.commit();
+ sdp.snapshot("c3");
+ writer.close();
+ long currentGen = 0;
+ for (IndexCommit ic : IndexReader.listCommits(dir)) {
+ assertTrue("currentGen=" + currentGen + " commitGen=" + ic.getGeneration(), currentGen < ic.getGeneration());
+ currentGen = ic.getGeneration();
+ }
+ dir.close();
+ }
+
}