You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2010/01/14 20:05:42 UTC
svn commit: r899359 [5/7] - in /lucene/java/branches/flex_1458: ./ contrib/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/
contrib/analyzers/common/src/java/org/apache/l...
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/SimpleAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -18,25 +18,15 @@
*/
import java.io.Reader;
-import java.io.IOException;
/** An {@link Analyzer} that filters {@link LetterTokenizer}
* with {@link LowerCaseFilter} */
-public final class SimpleAnalyzer extends Analyzer {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseTokenizer(reader);
- }
+public final class SimpleAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
- if (tokenizer == null) {
- tokenizer = new LowerCaseTokenizer(reader);
- setPreviousTokenStream(tokenizer);
- } else
- tokenizer.reset(reader);
- return tokenizer;
+ protected TokenStreamComponents createComponents(final String fieldName,
+ final Reader reader) {
+ return new TokenStreamComponents(new LowerCaseTokenizer(reader));
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -24,6 +24,7 @@
import java.util.Set;
import java.util.List;
+import org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents; // javadoc @link
import org.apache.lucene.util.Version;
/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}.
@@ -38,9 +39,7 @@
* </ul>
*/
-public final class StopAnalyzer extends Analyzer {
- private final Set<?> stopWords;
- private final Version matchVersion;
+public final class StopAnalyzer extends StopwordAnalyzerBase {
/** An unmodifiable set containing some common English words that are not usually useful
for searching.*/
@@ -65,16 +64,14 @@
* @param matchVersion See <a href="#version">above</a>
*/
public StopAnalyzer(Version matchVersion) {
- stopWords = ENGLISH_STOP_WORDS_SET;
- this.matchVersion = matchVersion;
+ this(matchVersion, ENGLISH_STOP_WORDS_SET);
}
/** Builds an analyzer with the stop words from the given set.
* @param matchVersion See <a href="#version">above</a>
* @param stopWords Set of stop words */
public StopAnalyzer(Version matchVersion, Set<?> stopWords) {
- this.stopWords = stopWords;
- this.matchVersion = matchVersion;
+ super(matchVersion, stopWords);
}
/** Builds an analyzer with the stop words from the given file.
@@ -82,8 +79,7 @@
* @param matchVersion See <a href="#version">above</a>
* @param stopwordsFile File to load stop words from */
public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException {
- stopWords = WordlistLoader.getWordSet(stopwordsFile);
- this.matchVersion = matchVersion;
+ this(matchVersion, WordlistLoader.getWordSet(stopwordsFile));
}
/** Builds an analyzer with the stop words from the given reader.
@@ -91,34 +87,21 @@
* @param matchVersion See <a href="#version">above</a>
* @param stopwords Reader to load stop words from */
public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
- stopWords = WordlistLoader.getWordSet(stopwords);
- this.matchVersion = matchVersion;
+ this(matchVersion, WordlistLoader.getWordSet(stopwords));
}
- /** Filters LowerCaseTokenizer with StopFilter. */
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new StopFilter(matchVersion,
- new LowerCaseTokenizer(reader), stopWords);
- }
-
- /** Filters LowerCaseTokenizer with StopFilter. */
- private class SavedStreams {
- Tokenizer source;
- TokenStream result;
- };
+ /**
+ * Creates {@link TokenStreamComponents} used to tokenize all the text in the provided {@link Reader}.
+ *
+ * @return {@link TokenStreamComponents} built from a {@link LowerCaseTokenizer} filtered with
+ * {@link StopFilter}
+ */
@Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- SavedStreams streams = (SavedStreams) getPreviousTokenStream();
- if (streams == null) {
- streams = new SavedStreams();
- streams.source = new LowerCaseTokenizer(reader);
- streams.result = new StopFilter(matchVersion,
- streams.source, stopWords);
- setPreviousTokenStream(streams);
- } else
- streams.source.reset(reader);
- return streams.result;
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ final Tokenizer source = new LowerCaseTokenizer(reader);
+ return new TokenStreamComponents(source, new StopFilter(matchVersion,
+ source, stopwords));
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/StopFilter.java Thu Jan 14 19:05:12 2010
@@ -64,6 +64,7 @@
* @param ignoreCase if true, all words are lower cased first
* @deprecated use {@link #StopFilter(Version, TokenStream, Set, boolean)} instead
*/
+ @Deprecated
public StopFilter(boolean enablePositionIncrements, TokenStream input, Set<?> stopWords, boolean ignoreCase)
{
this(Version.LUCENE_30, enablePositionIncrements, input, stopWords, ignoreCase);
@@ -101,7 +102,7 @@
*/
private StopFilter(Version matchVersion, boolean enablePositionIncrements, TokenStream input, Set<?> stopWords, boolean ignoreCase){
super(input);
- this.stopWords = CharArraySet.unmodifiableSet(new CharArraySet(matchVersion, stopWords, ignoreCase));
+ this.stopWords = stopWords instanceof CharArraySet ? (CharArraySet)stopWords : new CharArraySet(matchVersion, stopWords, ignoreCase);
this.enablePositionIncrements = enablePositionIncrements;
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
@@ -117,6 +118,7 @@
* @see #makeStopSet(Version, java.lang.String[])
* @deprecated use {@link #StopFilter(Version, TokenStream, Set)} instead
*/
+ @Deprecated
public StopFilter(boolean enablePositionIncrements, TokenStream in, Set<?> stopWords) {
this(Version.LUCENE_CURRENT, enablePositionIncrements, in, stopWords, false);
}
@@ -148,6 +150,7 @@
* @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
* @deprecated use {@link #makeStopSet(Version, String...)} instead
*/
+ @Deprecated
public static final Set<Object> makeStopSet(String... stopWords) {
return makeStopSet(Version.LUCENE_30, stopWords, false);
}
@@ -176,6 +179,7 @@
* @see #makeStopSet(Version, java.lang.String[], boolean) passing false to ignoreCase
* @deprecated use {@link #makeStopSet(Version, List)} instead
*/
+ @Deprecated
public static final Set<Object> makeStopSet(List<?> stopWords) {
return makeStopSet(Version.LUCENE_30, stopWords, false);
}
@@ -202,6 +206,7 @@
* @return a Set containing the words
* @deprecated use {@link #makeStopSet(Version, String[], boolean)} instead;
*/
+ @Deprecated
public static final Set<Object> makeStopSet(String[] stopWords, boolean ignoreCase) {
return makeStopSet(Version.LUCENE_30, stopWords, ignoreCase);
}
@@ -226,6 +231,7 @@
* @return A Set ({@link CharArraySet}) containing the words
* @deprecated use {@link #makeStopSet(Version, List, boolean)} instead
*/
+ @Deprecated
public static final Set<Object> makeStopSet(List<?> stopWords, boolean ignoreCase){
return makeStopSet(Version.LUCENE_30, stopWords, ignoreCase);
}
@@ -271,6 +277,7 @@
* or later, it returns true.
* @deprecated use {@link #StopFilter(Version, TokenStream, Set)} instead
*/
+ @Deprecated
public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) {
return matchVersion.onOrAfter(Version.LUCENE_29);
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java Thu Jan 14 19:05:12 2010
@@ -29,8 +29,6 @@
NOTE: Subclasses overriding {@link #incrementToken()} must
call {@link AttributeSource#clearAttributes()} before
setting attributes.
- Subclasses overriding {@link #incrementToken()} must call
- {@link Token#clear()} before setting Token attributes.
*/
public abstract class Tokenizer extends TokenStream {
/** The text source for this Tokenizer. */
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -18,24 +18,14 @@
*/
import java.io.Reader;
-import java.io.IOException;
/** An Analyzer that uses {@link WhitespaceTokenizer}. */
-public final class WhitespaceAnalyzer extends Analyzer {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new WhitespaceTokenizer(reader);
- }
+public final class WhitespaceAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
- if (tokenizer == null) {
- tokenizer = new WhitespaceTokenizer(reader);
- setPreviousTokenStream(tokenizer);
- } else
- tokenizer.reset(reader);
- return tokenizer;
+ protected TokenStreamComponents createComponents(final String fieldName,
+ final Reader reader) {
+ return new TokenStreamComponents(new WhitespaceTokenizer(reader));
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/WordlistLoader.java Thu Jan 14 19:05:12 2010
@@ -21,15 +21,69 @@
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Set;
/**
* Loader for text files that represent a list of stopwords.
*/
public class WordlistLoader {
-
+
+ /**
+ * Loads a text file associated with a given class (See
+ * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
+ * to a {@link Set} (omitting leading and trailing whitespace). Every line of
+ * the file should contain only one word. The words need to be in lower-case if
+ * you make use of an Analyzer which uses LowerCaseFilter (like
+ * StandardAnalyzer).
+ *
+ * @param aClass
+ * a class that is associated with the given stopwordResource
+ * @param stopwordResource
+ * name of the resource file associated with the given class
+ * @return a {@link Set} with the file's words
+ */
+ public static Set<String> getWordSet(Class<?> aClass, String stopwordResource)
+ throws IOException {
+ final Reader reader = new BufferedReader(new InputStreamReader(aClass
+ .getResourceAsStream(stopwordResource), "UTF-8"));
+ try {
+ return getWordSet(reader);
+ } finally {
+ reader.close();
+ }
+ }
+
+ /**
+ * Loads a text file associated with a given class (See
+ * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
+ * to a {@link Set} (omitting leading and trailing whitespace). Every line of
+ * the file should contain only one word. The words need to be in lower-case if
+ * you make use of an Analyzer which uses LowerCaseFilter (like
+ * StandardAnalyzer).
+ *
+ * @param aClass
+ * a class that is associated with the given stopwordResource
+ * @param stopwordResource
+ * name of the resource file associated with the given class
+ * @param comment
+ * the comment string to ignore
+ * @return a {@link Set} with the file's words
+ */
+ public static Set<String> getWordSet(Class<?> aClass,
+ String stopwordResource, String comment) throws IOException {
+ final Reader reader = new BufferedReader(new InputStreamReader(aClass
+ .getResourceAsStream(stopwordResource), "UTF-8"));
+ try {
+ return getWordSet(reader, comment);
+ } finally {
+ reader.close();
+ }
+ }
+
/**
* Loads a text file and adds every line as an entry to a HashSet (omitting
* leading and trailing whitespace). Every line of the file should contain only
@@ -40,17 +94,15 @@
* @return A HashSet with the file's words
*/
public static HashSet<String> getWordSet(File wordfile) throws IOException {
- HashSet<String> result = new HashSet<String>();
FileReader reader = null;
try {
reader = new FileReader(wordfile);
- result = getWordSet(reader);
+ return getWordSet(reader);
}
finally {
if (reader != null)
reader.close();
}
- return result;
}
/**
@@ -64,17 +116,15 @@
* @return A HashSet with the file's words
*/
public static HashSet<String> getWordSet(File wordfile, String comment) throws IOException {
- HashSet<String> result = new HashSet<String>();
FileReader reader = null;
try {
reader = new FileReader(wordfile);
- result = getWordSet(reader, comment);
+ return getWordSet(reader, comment);
}
finally {
if (reader != null)
reader.close();
}
- return result;
}
@@ -88,7 +138,7 @@
* @return A HashSet with the reader's words
*/
public static HashSet<String> getWordSet(Reader reader) throws IOException {
- HashSet<String> result = new HashSet<String>();
+ final HashSet<String> result = new HashSet<String>();
BufferedReader br = null;
try {
if (reader instanceof BufferedReader) {
@@ -119,7 +169,7 @@
* @return A HashSet with the reader's words
*/
public static HashSet<String> getWordSet(Reader reader, String comment) throws IOException {
- HashSet<String> result = new HashSet<String>();
+ final HashSet<String> result = new HashSet<String>();
BufferedReader br = null;
try {
if (reader instanceof BufferedReader) {
@@ -146,7 +196,7 @@
/**
* Reads a stem dictionary. Each line contains:
* <pre>word<b>\t</b>stem</pre>
- * (i.e. two tab seperated words)
+ * (i.e. two tab separated words)
*
* @return stem dictionary that overrules the stemming algorithm
* @throws IOException
@@ -154,21 +204,18 @@
public static HashMap<String, String> getStemDict(File wordstemfile) throws IOException {
if (wordstemfile == null)
throw new NullPointerException("wordstemfile may not be null");
- HashMap<String, String> result = new HashMap<String, String>();
+ final HashMap<String, String> result = new HashMap<String,String>();
BufferedReader br = null;
- FileReader fr = null;
+
try {
- fr = new FileReader(wordstemfile);
- br = new BufferedReader(fr);
+ br = new BufferedReader(new FileReader(wordstemfile));
String line;
while ((line = br.readLine()) != null) {
String[] wordstem = line.split("\t", 2);
result.put(wordstem[0], wordstem[1]);
}
} finally {
- if (fr != null)
- fr.close();
- if (br != null)
+ if(br != null)
br.close();
}
return result;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Thu Jan 14 19:05:12 2010
@@ -71,7 +71,6 @@
* @param stopWords stop words */
public StandardAnalyzer(Version matchVersion, Set<?> stopWords) {
stopSet = stopWords;
- setOverridesTokenStreamMethod(StandardAnalyzer.class);
replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24);
this.matchVersion = matchVersion;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Thu Jan 14 19:05:12 2010
@@ -71,6 +71,7 @@
* @deprecated this solves a bug where HOSTs that end with '.' are identified
* as ACRONYMs.
*/
+ @Deprecated
public static final int ACRONYM_DEP = 8;
/** String token types that correspond to token type int constants */
@@ -227,6 +228,7 @@
*
* @deprecated Remove in 3.X and make true the only valid value
*/
+ @Deprecated
public boolean isReplaceInvalidAcronym() {
return replaceInvalidAcronym;
}
@@ -238,6 +240,7 @@
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*/
+ @Deprecated
public void setReplaceInvalidAcronym(boolean replaceInvalidAcronym) {
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Thu Jan 14 19:05:12 2010
@@ -354,6 +354,7 @@
* @deprecated this solves a bug where HOSTs that end with '.' are identified
* as ACRONYMs.
*/
+@Deprecated
public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Thu Jan 14 19:05:12 2010
@@ -55,6 +55,7 @@
* @deprecated this solves a bug where HOSTs that end with '.' are identified
* as ACRONYMs.
*/
+@Deprecated
public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/collation/CollationKeyFilter.java Thu Jan 14 19:05:12 2010
@@ -24,8 +24,6 @@
import org.apache.lucene.util.IndexableBinaryStringTools;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
import java.text.Collator;
@@ -94,15 +92,14 @@
char[] termBuffer = termAtt.termBuffer();
String termText = new String(termBuffer, 0, termAtt.termLength());
byte[] collationKey = collator.getCollationKey(termText).toByteArray();
- ByteBuffer collationKeyBuf = ByteBuffer.wrap(collationKey);
- int encodedLength
- = IndexableBinaryStringTools.getEncodedLength(collationKeyBuf);
+ int encodedLength = IndexableBinaryStringTools.getEncodedLength(
+ collationKey, 0, collationKey.length);
if (encodedLength > termBuffer.length) {
termAtt.resizeTermBuffer(encodedLength);
}
termAtt.setTermLength(encodedLength);
- CharBuffer wrappedTermBuffer = CharBuffer.wrap(termAtt.termBuffer());
- IndexableBinaryStringTools.encode(collationKeyBuf, wrappedTermBuffer);
+ IndexableBinaryStringTools.encode(collationKey, 0, collationKey.length,
+ termAtt.termBuffer(), 0, encodedLength);
return true;
} else {
return false;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/DateField.java Thu Jan 14 19:05:12 2010
@@ -57,6 +57,7 @@
* This class is included for use with existing
* indices and will be removed in a future release (possibly Lucene 4.0).
*/
+@Deprecated
public class DateField {
private DateField() {}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/Field.java Thu Jan 14 19:05:12 2010
@@ -521,11 +521,27 @@
*
* @param name The name of the field
* @param value The binary value
- * @param store How <code>value</code> should be stored (compressed or not)
+ * @param store Must be Store.YES
* @throws IllegalArgumentException if store is <code>Store.NO</code>
+ * @deprecated Use {@link #Field(String, byte[]) instead}
*/
+ @Deprecated
public Field(String name, byte[] value, Store store) {
- this(name, value, 0, value.length, store);
+ this(name, value, 0, value.length);
+
+ if (store == Store.NO) {
+ throw new IllegalArgumentException("binary values can't be unstored");
+ }
+ }
+
+ /**
+ * Create a stored field with binary value. Optionally the value may be compressed.
+ *
+ * @param name The name of the field
+ * @param value The binary value
+ */
+ public Field(String name, byte[] value) {
+ this(name, value, 0, value.length);
}
/**
@@ -537,8 +553,26 @@
* @param length Number of bytes to use for this Field, starting at offset
* @param store How <code>value</code> should be stored (compressed or not)
* @throws IllegalArgumentException if store is <code>Store.NO</code>
+ * @deprecated Use {@link #Field(String, byte[], int, int) instead}
*/
+ @Deprecated
public Field(String name, byte[] value, int offset, int length, Store store) {
+ this(name, value, offset, length);
+
+ if (store == Store.NO) {
+ throw new IllegalArgumentException("binary values can't be unstored");
+ }
+ }
+
+ /**
+ * Create a stored field with binary value. Optionally the value may be compressed.
+ *
+ * @param name The name of the field
+ * @param value The binary value
+ * @param offset Starting offset in value where this Field's bytes are
+ * @param length Number of bytes to use for this Field, starting at offset
+ */
+ public Field(String name, byte[] value, int offset, int length) {
if (name == null)
throw new IllegalArgumentException("name cannot be null");
@@ -548,10 +582,7 @@
this.name = StringHelper.intern(name); // field names are interned
fieldsData = value;
- if (store == Store.NO)
- throw new IllegalArgumentException("binary values can't be unstored");
-
- isStored = store.isStored();
+ isStored = true;
isIndexed = false;
isTokenized = false;
omitTermFreqAndPositions = false;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/document/NumberTools.java Thu Jan 14 19:05:12 2010
@@ -44,6 +44,7 @@
* This class is included for use with existing
* indices and will be removed in a future release (possibly Lucene 4.0).
*/
+@Deprecated
public class NumberTools {
private static final int RADIX = 36;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java Thu Jan 14 19:05:12 2010
@@ -31,6 +31,7 @@
}
}
+ @Override
public boolean isDeleted(int doc) {
return deletedDocs != null && deletedDocs.get(doc);
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Thu Jan 14 19:05:12 2010
@@ -23,24 +23,45 @@
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Collections;
/** A {@link MergeScheduler} that runs each merge using a
- * separate thread, up until a maximum number of threads
- * ({@link #setMaxThreadCount}) at which when a merge is
- * needed, the thread(s) that are updating the index will
- * pause until one or more merges completes. This is a
- * simple way to use concurrency in the indexing process
- * without having to create and manage application level
- * threads. */
-
+ * separate thread.
+ *
+ * <p>Specify the max number of threads that may run at
+ * once with {@link #setMaxThreadCount}.</p>
+ *
+ * <p>Separately specify the maximum number of simultaneous
+ * merges with {@link #setMaxMergeCount}. If the number of
+ * merges exceeds the max number of threads then the
+ * largest merges are paused until one of the smaller
+ * merges completes.</p>
+ *
+ * <p>If more than {@link #getMaxMergeCount} merges are
+ * requested then this class will forcefully throttle the
+ * incoming threads by pausing until one more more merges
+ * complete.</p>
+ */
public class ConcurrentMergeScheduler extends MergeScheduler {
private int mergeThreadPriority = -1;
protected List<MergeThread> mergeThreads = new ArrayList<MergeThread>();
- // Max number of threads allowed to be merging at once
- private int maxThreadCount = 1;
+ // Max number of merge threads allowed to be running at
+ // once. When there are more merges then this, we
+ // forcefully pause the larger ones, letting the smaller
+ // ones run, up until maxMergeCount merges at which point
+ // we forcefully pause incoming threads (that presumably
+ // are the ones causing so much merging). We dynamically
+ // default this from 1 to 3, depending on how many cores
+ // you have:
+ private int maxThreadCount = Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors()/2));
+
+ // Max number of merges we accept before forcefully
+ // throttling the incoming threads
+ private int maxMergeCount = maxThreadCount+2;
protected Directory dir;
@@ -55,23 +76,45 @@
}
}
- /** Sets the max # simultaneous threads that may be
- * running. If a merge is necessary yet we already have
- * this many threads running, the incoming thread (that
- * is calling add/updateDocument) will block until
- * a merge thread has completed. */
+ /** Sets the max # simultaneous merge threads that should
+ * be running at once. This must be <= {@link
+ * #setMaxMergeCount}. */
public void setMaxThreadCount(int count) {
- if (count < 1)
+ if (count < 1) {
throw new IllegalArgumentException("count should be at least 1");
+ }
+ if (count > maxMergeCount) {
+ throw new IllegalArgumentException("count should be <= maxMergeCount (= " + maxMergeCount + ")");
+ }
maxThreadCount = count;
}
- /** Get the max # simultaneous threads that may be
- * running. @see #setMaxThreadCount. */
+ /** @see #setMaxThreadCount(int) */
public int getMaxThreadCount() {
return maxThreadCount;
}
+ /** Sets the max # simultaneous merges that are allowed.
+ * If a merge is necessary yet we already have this many
+ * threads running, the incoming thread (that is calling
+ * add/updateDocument) will block until a merge thread
+ * has completed. Note that we will only run the
+ * smallest {@link #setMaxThreadCount} merges at a time. */
+ public void setMaxMergeCount(int count) {
+ if (count < 1) {
+ throw new IllegalArgumentException("count should be at least 1");
+ }
+ if (count < maxThreadCount) {
+ throw new IllegalArgumentException("count should be >= maxThreadCount (= " + maxThreadCount + ")");
+ }
+ maxMergeCount = count;
+ }
+
+ /** See {@link #setMaxMergeCount}. */
+ public int getMaxMergeCount() {
+ return maxMergeCount;
+ }
+
/** Return the priority that merge threads run at. By
* default the priority is 1 plus the priority of (ie,
* slightly higher priority than) the first thread that
@@ -81,16 +124,73 @@
return mergeThreadPriority;
}
- /** Set the priority that merge threads run at. */
+ /** Set the base priority that merge threads run at.
+ * Note that CMS may increase priority of some merge
+ * threads beyond this base priority. It's best not to
+ * set this any higher than
+ * Thread.MAX_PRIORITY-maxThreadCount, so that CMS has
+ * room to set relative priority among threads. */
public synchronized void setMergeThreadPriority(int pri) {
if (pri > Thread.MAX_PRIORITY || pri < Thread.MIN_PRIORITY)
throw new IllegalArgumentException("priority must be in range " + Thread.MIN_PRIORITY + " .. " + Thread.MAX_PRIORITY + " inclusive");
mergeThreadPriority = pri;
+ updateMergeThreads();
+ }
- final int numThreads = mergeThreadCount();
- for(int i=0;i<numThreads;i++) {
- MergeThread merge = mergeThreads.get(i);
- merge.setThreadPriority(pri);
+ // Larger merges come first
+ protected static class CompareByMergeDocCount implements Comparator<MergeThread> {
+ public int compare(MergeThread t1, MergeThread t2) {
+ final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
+ final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
+
+ final int c1 = m1 == null ? Integer.MAX_VALUE : m1.segments.totalDocCount();
+ final int c2 = m2 == null ? Integer.MAX_VALUE : m2.segments.totalDocCount();
+
+ return c2 - c1;
+ }
+ }
+
+ /** Called whenever the running merges have changed, to
+ * pause & unpause threads. */
+ protected synchronized void updateMergeThreads() {
+
+ Collections.sort(mergeThreads, new CompareByMergeDocCount());
+
+ final int count = mergeThreads.size();
+ int pri = mergeThreadPriority;
+ for(int i=0;i<count;i++) {
+ final MergeThread mergeThread = mergeThreads.get(i);
+ final MergePolicy.OneMerge merge = mergeThread.getCurrentMerge();
+ if (merge == null) {
+ continue;
+ }
+ final boolean doPause;
+ if (i < count-maxThreadCount) {
+ doPause = true;
+ } else {
+ doPause = false;
+ }
+
+ if (verbose()) {
+ if (doPause != merge.getPause()) {
+ if (doPause) {
+ message("pause thread " + mergeThread.getName());
+ } else {
+ message("unpause thread " + mergeThread.getName());
+ }
+ }
+ }
+ if (doPause != merge.getPause()) {
+ merge.setPause(doPause);
+ }
+
+ if (!doPause) {
+ if (verbose()) {
+ message("set priority of merge thread " + mergeThread.getName() + " to " + pri);
+ }
+ mergeThread.setThreadPriority(pri);
+ pri = Math.min(Thread.MAX_PRIORITY, 1+pri);
+ }
}
}
@@ -192,9 +292,12 @@
try {
synchronized(this) {
final MergeThread merger;
- while (mergeThreadCount() >= maxThreadCount) {
- if (verbose())
- message(" too many merge threads running; stalling...");
+ long startStallTime = 0;
+ while (mergeThreadCount() >= maxMergeCount) {
+ startStallTime = System.currentTimeMillis();
+ if (verbose()) {
+ message(" too many merges; stalling...");
+ }
try {
wait();
} catch (InterruptedException ie) {
@@ -202,15 +305,20 @@
}
}
- if (verbose())
+ if (verbose()) {
+ if (startStallTime != 0) {
+ message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
+ }
message(" consider merge " + merge.segString(dir));
-
- assert mergeThreadCount() < maxThreadCount;
+ }
+
+ assert mergeThreadCount() < maxMergeCount;
// OK to spawn a new merge thread to handle this
// merge:
merger = getMergeThread(writer, merge);
mergeThreads.add(merger);
+ updateMergeThreads();
if (verbose())
message(" launch new thread [" + merger.getName() + "]");
@@ -245,6 +353,7 @@
IndexWriter writer;
MergePolicy.OneMerge startMerge;
MergePolicy.OneMerge runningMerge;
+ private volatile boolean done;
public MergeThread(IndexWriter writer, MergePolicy.OneMerge startMerge) throws IOException {
this.writer = writer;
@@ -259,6 +368,16 @@
return runningMerge;
}
+ public synchronized MergePolicy.OneMerge getCurrentMerge() {
+ if (done) {
+ return null;
+ } else if (runningMerge != null) {
+ return runningMerge;
+ } else {
+ return startMerge;
+ }
+ }
+
public void setThreadPriority(int pri) {
try {
setPriority(pri);
@@ -292,10 +411,14 @@
merge = writer.getNextMerge();
if (merge != null) {
writer.mergeInit(merge);
+ updateMergeThreads();
if (verbose())
message(" merge thread: do another merge " + merge.segString(dir));
- } else
+ } else {
+ done = true;
+ updateMergeThreads();
break;
+ }
}
if (verbose())
@@ -317,6 +440,7 @@
ConcurrentMergeScheduler.this.notifyAll();
boolean removed = mergeThreads.remove(this);
assert removed;
+ updateMergeThreads();
}
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Thu Jan 14 19:05:12 2010
@@ -32,7 +32,6 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.search.Similarity;
-import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
@@ -41,6 +40,8 @@
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
+
/**
* An IndexReader which reads indexes with multiple segments.
*/
@@ -154,7 +155,7 @@
DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
- this.segmentInfos = infos;
+ segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (codecs == null) {
this.codecs = Codecs.getDefault();
@@ -340,6 +341,22 @@
}
}
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder();
+ if (hasChanges) {
+ buffer.append("*");
+ }
+ buffer.append(getClass().getSimpleName());
+ buffer.append('(');
+ for(SegmentReader r : subReaders) {
+ buffer.append(r);
+ }
+ buffer.append(')');
+ return buffer.toString();
+ }
+
private void initialize(SegmentReader[] subReaders) {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
@@ -456,22 +473,39 @@
}
@Override
- public final synchronized IndexReader reopen() throws CorruptIndexException, IOException {
+ public final IndexReader reopen() throws CorruptIndexException, IOException {
// Preserve current readOnly
return doReopen(readOnly, null);
}
@Override
- public final synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
+ public final IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
return doReopen(openReadOnly, null);
}
@Override
- public final synchronized IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException {
+ public final IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException {
return doReopen(true, commit);
}
- private synchronized IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
+ private final IndexReader doReopenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
+ assert readOnly;
+
+ if (!openReadOnly) {
+ throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
+ }
+
+ if (commit != null) {
+ throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
+ }
+
+ // TODO: right now we *always* make a new reader; in
+ // the future we could have write make some effort to
+ // detect that no changes have occurred
+ return writer.getReader();
+ }
+
+ private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
ensureOpen();
assert commit == null || openReadOnly;
@@ -479,26 +513,13 @@
// If we were obtained by writer.getReader(), re-ask the
// writer to get a new reader.
if (writer != null) {
- assert readOnly;
-
- if (!openReadOnly) {
- throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
- }
-
- if (commit != null) {
- throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
- }
-
- if (!writer.isOpen(true)) {
- throw new AlreadyClosedException("cannot reopen: the IndexWriter this reader was obtained from is now closed");
- }
-
- // TODO: right now we *always* make a new reader; in
- // the future we could have write make some effort to
- // detect that no changes have occurred
- IndexReader reader = writer.getReader();
- return reader;
+ return doReopenFromWriter(openReadOnly, commit);
+ } else {
+ return doReopenNoWriter(openReadOnly, commit);
}
+ }
+
+ private synchronized IndexReader doReopenNoWriter(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
if (commit == null) {
if (hasChanges) {
@@ -602,10 +623,13 @@
ensureOpen();
return segmentInfos.size() == 1 && !hasDeletions();
}
-
+
@Override
- public synchronized int numDocs() {
+ public int numDocs() {
// Don't call ensureOpen() here (it could affect performance)
+
+ // NOTE: multiple threads may wind up init'ing
+ // numDocs... but that's harmless
if (numDocs == -1) { // check cache
int n = 0; // cache miss--recompute
for (int i = 0; i < subReaders.length; i++)
@@ -991,6 +1015,12 @@
if (ioe == null) ioe = e;
}
}
+
+ // NOTE: only needed in case someone had asked for
+ // FieldCache for top-level reader (which is generally
+ // not a good idea):
+ FieldCache.DEFAULT.purge(this);
+
// throw the first exception
if (ioe != null) throw ioe;
}
@@ -1025,6 +1055,7 @@
return directory;
}
+ @Override
public int getTermInfosIndexDivisor() {
return termInfosIndexDivisor;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Thu Jan 14 19:05:12 2010
@@ -245,8 +245,10 @@
for(int i=0;i<fieldCount;i++)
fields[i].consumer.processFields(fields[i].fields, fields[i].fieldCount);
- if (docState.maxTermPrefix != null && docState.infoStream != null)
+ if (docState.maxTermPrefix != null && docState.infoStream != null) {
docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
+ docState.maxTermPrefix = null;
+ }
final DocumentsWriter.DocWriter one = fieldsWriter.finishDocument();
final DocumentsWriter.DocWriter two = consumer.finishDocument();
@@ -386,4 +388,4 @@
}
}
}
-}
+}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsReader.java Thu Jan 14 19:05:12 2010
@@ -164,7 +164,7 @@
/**
* @throws AlreadyClosedException if this FieldsReader is closed
*/
- protected final void ensureOpen() throws AlreadyClosedException {
+ private void ensureOpen() throws AlreadyClosedException {
if (closed) {
throw new AlreadyClosedException("this FieldsReader is closed");
}
@@ -412,6 +412,7 @@
private int toRead;
private long pointer;
/** @deprecated Only kept for backward-compatbility with <3.0 indexes. Will be removed in 4.0. */
+ @Deprecated
private boolean isCompressed;
public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary, boolean isCompressed) {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsWriter.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
static final byte FIELD_IS_BINARY = 0x2;
/** @deprecated Kept for backwards-compatibility with <3.0 indexes; will be removed in 4.0 */
+ @Deprecated
static final byte FIELD_IS_COMPRESSED = 0x4;
// Original format
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java Thu Jan 14 19:05:12 2010
@@ -21,6 +21,7 @@
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import java.io.IOException;
import java.util.Collection;
@@ -253,7 +254,14 @@
protected void doCommit(Map<String,String> commitUserData) throws IOException { in.commit(commitUserData); }
@Override
- protected void doClose() throws IOException { in.close(); }
+ protected void doClose() throws IOException {
+ in.close();
+
+ // NOTE: only needed in case someone had asked for
+ // FieldCache for top-level reader (which is generally
+ // not a good idea):
+ FieldCache.DEFAULT.purge(this);
+ }
@Override
@@ -284,4 +292,21 @@
public IndexReader[] getSequentialSubReaders() {
return in.getSequentialSubReaders();
}
-}
+
+ /** If the subclass of FilteredIndexReader modifies the
+ * contents of the FieldCache, you must override this
+ * method to provide a different key */
+ @Override
+ public Object getFieldCacheKey() {
+ return in.getFieldCacheKey();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder("FilterReader(");
+ buffer.append(in);
+ buffer.append(')');
+ return buffer.toString();
+ }
+}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java Thu Jan 14 19:05:12 2010
@@ -537,7 +537,7 @@
// the file for subsequent deletion.
if (infoStream != null) {
- message("IndexFileDeleter: unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
+ message("unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
}
if (deletable == null) {
deletable = new ArrayList<String>();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
/** IndexReader is an abstract class, providing an interface for accessing an
index. Search of an index is done entirely through this abstract interface,
@@ -119,13 +120,13 @@
private boolean closed;
protected boolean hasChanges;
- private int refCount;
+ private final AtomicInteger refCount = new AtomicInteger();
static int DEFAULT_TERMS_INDEX_DIVISOR = 1;
/** Expert: returns the current refCount for this reader */
- public synchronized int getRefCount() {
- return refCount;
+ public int getRefCount() {
+ return refCount.get();
}
/**
@@ -142,41 +143,68 @@
*
* @see #decRef
*/
- public synchronized void incRef() {
- assert refCount > 0;
+ public void incRef() {
ensureOpen();
- refCount++;
+ refCount.incrementAndGet();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder();
+ if (hasChanges) {
+ buffer.append('*');
+ }
+ buffer.append(getClass().getSimpleName());
+ buffer.append('(');
+ final IndexReader[] subReaders = getSequentialSubReaders();
+ if ((subReaders != null) && (subReaders.length > 0)) {
+ buffer.append(subReaders[0]);
+ for (int i = 1; i < subReaders.length; ++i) {
+ buffer.append(" ").append(subReaders[i]);
+ }
+ }
+ buffer.append(')');
+ return buffer.toString();
}
/**
* Expert: decreases the refCount of this IndexReader
* instance. If the refCount drops to 0, then pending
* changes (if any) are committed to the index and this
- * reader is closed.
- *
+ * reader is closed. If an exception is hit, the refCount
+ * is unchanged.
+ *
* @throws IOException in case an IOException occurs in commit() or doClose()
*
* @see #incRef
*/
- public synchronized void decRef() throws IOException {
- assert refCount > 0;
+ public void decRef() throws IOException {
ensureOpen();
- if (refCount == 1) {
- commit();
- doClose();
+ if (refCount.getAndDecrement() == 1) {
+ boolean success = false;
+ try {
+ commit();
+ doClose();
+ success = true;
+ } finally {
+ if (!success) {
+ // Put reference back on failure
+ refCount.incrementAndGet();
+ }
+ }
}
- refCount--;
}
protected IndexReader() {
- refCount = 1;
+ refCount.set(1);
}
/**
* @throws AlreadyClosedException if this IndexReader is closed
*/
protected final void ensureOpen() throws AlreadyClosedException {
- if (refCount <= 0) {
+ if (refCount.get() <= 0) {
throw new AlreadyClosedException("this IndexReader is closed");
}
}
@@ -1362,4 +1390,13 @@
}
return numTerms;
}
+
+ /** For IndexReader implementations that use
+ * TermInfosReader to read terms, this returns the
+ * current indexDivisor as specified when the reader was
+ * opened.
+ */
+ public int getTermInfosIndexDivisor() {
+ throw new UnsupportedOperationException("This reader does not support this method.");
+ }
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java Thu Jan 14 19:05:12 2010
@@ -388,6 +388,9 @@
* loading a TermInfo. The default value is 1. Set this
* to -1 to skip loading the terms index entirely. */
public IndexReader getReader(int termInfosIndexDivisor) throws IOException {
+
+ ensureOpen();
+
if (infoStream != null) {
message("flush at getReader");
}
@@ -397,12 +400,13 @@
// this method is called:
poolReaders = true;
- flush(true, true, true);
+ flush(true, true, false);
// Prevent segmentInfos from changing while opening the
// reader; in theory we could do similar retry logic,
// just like we do when loading segments_N
synchronized(this) {
+ applyDeletes();
return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor, codecs);
}
}
@@ -706,23 +710,19 @@
notifyAll();
}
- synchronized final boolean isOpen(boolean includePendingClose) {
- return !(closed || (includePendingClose && closing));
- }
-
/**
* Used internally to throw an {@link
* AlreadyClosedException} if this IndexWriter has been
* closed.
* @throws AlreadyClosedException if this IndexWriter is
*/
- protected synchronized final void ensureOpen(boolean includePendingClose) throws AlreadyClosedException {
- if (!isOpen(includePendingClose)) {
+ protected final void ensureOpen(boolean includePendingClose) throws AlreadyClosedException {
+ if (closed || (includePendingClose && closing)) {
throw new AlreadyClosedException("this IndexWriter is closed");
}
}
- protected synchronized final void ensureOpen() throws AlreadyClosedException {
+ protected final void ensureOpen() throws AlreadyClosedException {
ensureOpen(true);
}
@@ -1502,7 +1502,8 @@
" maxBufferedDocs=" + docWriter.getMaxBufferedDocs() +
" maxBuffereDeleteTerms=" + docWriter.getMaxBufferedDeleteTerms() +
" maxFieldLength=" + maxFieldLength +
- " index=" + segString());
+ " index=" + segString() +
+ " version=" + Constants.LUCENE_VERSION);
}
/**
@@ -2918,7 +2919,7 @@
releaseWrite();
}
- private void blockAddIndexes(boolean includePendingClose) {
+ private void blockAddIndexes() {
acquireRead();
@@ -2927,7 +2928,7 @@
// Make sure we are still open since we could have
// waited quite a while for last addIndexes to finish
- ensureOpen(includePendingClose);
+ ensureOpen(false);
success = true;
} finally {
if (!success)
@@ -3668,7 +3669,6 @@
}
if (flushDeletes) {
- flushDeletesCount++;
applyDeletes();
}
@@ -3962,7 +3962,7 @@
handleOOM(oom, "merge");
}
if (infoStream != null) {
- message("merge time " + (System.currentTimeMillis()-t0) + " msec");
+ message("merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
}
}
@@ -4453,6 +4453,7 @@
// Apply buffered deletes to all segments.
private final synchronized boolean applyDeletes() throws CorruptIndexException, IOException {
assert testPoint("startApplyDeletes");
+ flushDeletesCount++;
SegmentInfos rollback = (SegmentInfos) segmentInfos.clone();
boolean success = false;
boolean changed;
@@ -4514,7 +4515,7 @@
buffer.append(' ');
}
final SegmentInfo info = infos.info(i);
- buffer.append(info.segString(directory));
+ buffer.append(info.toString(directory, 0));
if (info.dir != directory)
buffer.append("**");
}
@@ -4618,7 +4619,7 @@
// Wait for any running addIndexes to complete
// first, then block any from running until we've
// copied the segmentInfos we intend to sync:
- blockAddIndexes(false);
+ blockAddIndexes();
// On commit the segmentInfos must never
// reference a segment in another directory:
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MergePolicy.java Thu Jan 14 19:05:12 2010
@@ -85,6 +85,7 @@
final boolean useCompoundFile;
boolean aborted;
Throwable error;
+ boolean paused;
public OneMerge(SegmentInfos segments, boolean useCompoundFile) {
if (0 == segments.size())
@@ -110,6 +111,7 @@
* not be committed. */
synchronized void abort() {
aborted = true;
+ notifyAll();
}
/** Returns true if this merge was aborted. */
@@ -118,8 +120,34 @@
}
synchronized void checkAborted(Directory dir) throws MergeAbortedException {
- if (aborted)
+ if (aborted) {
throw new MergeAbortedException("merge is aborted: " + segString(dir));
+ }
+
+ while (paused) {
+ try {
+ // In theory we could wait() indefinitely, but we
+ // do 1000 msec, defensively
+ wait(1000);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException(ie);
+ }
+ if (aborted) {
+ throw new MergeAbortedException("merge is aborted: " + segString(dir));
+ }
+ }
+ }
+
+ synchronized public void setPause(boolean paused) {
+ this.paused = paused;
+ if (!paused) {
+ // Wakeup merge thread, if it's waiting
+ notifyAll();
+ }
+ }
+
+ synchronized public boolean getPause() {
+ return paused;
}
String segString(Directory dir) {
@@ -127,7 +155,7 @@
final int numSegments = segments.size();
for(int i=0;i<numSegments;i++) {
if (i > 0) b.append(' ');
- b.append(segments.info(i).segString(dir));
+ b.append(segments.info(i).toString(dir, 0));
}
if (info != null)
b.append(" into ").append(info.name);
@@ -262,5 +290,4 @@
* compound file format.
*/
public abstract boolean useCompoundDocStore(SegmentInfos segments);
-
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java Thu Jan 14 19:05:12 2010
@@ -32,6 +32,7 @@
import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
/** An IndexReader which reads multiple indexes, appending
* their content. */
@@ -257,8 +258,10 @@
}
@Override
- public synchronized int numDocs() {
+ public int numDocs() {
// Don't call ensureOpen() here (it could affect performance)
+ // NOTE: multiple threads may wind up init'ing
+ // numDocs... but that's harmless
if (numDocs == -1) { // check cache
int n = 0; // cache miss--recompute
for (int i = 0; i < subReaders.length; i++)
@@ -459,6 +462,11 @@
subReaders[i].close();
}
}
+
+ // NOTE: only needed in case someone had asked for
+ // FieldCache for top-level reader (which is generally
+ // not a good idea):
+ FieldCache.DEFAULT.purge(this);
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ParallelReader.java Thu Jan 14 19:05:12 2010
@@ -22,6 +22,7 @@
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import java.io.IOException;
import java.util.*;
@@ -72,6 +73,21 @@
this.incRefReaders = !closeSubReaders;
}
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder("ParallelReader(");
+ final Iterator<IndexReader> iter = readers.iterator();
+ if (iter.hasNext()) {
+ buffer.append(iter.next());
+ }
+ while (iter.hasNext()) {
+ buffer.append(", ").append(iter.next());
+ }
+ buffer.append(')');
+ return buffer.toString();
+ }
+
/** Add an IndexReader.
* @throws IOException if there is a low-level IO error
*/
@@ -544,6 +560,8 @@
readers.get(i).close();
}
}
+
+ FieldCache.DEFAULT.purge(this);
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java Thu Jan 14 19:05:12 2010
@@ -98,11 +98,6 @@
private Map<String,String> diagnostics;
- @Override
- public String toString() {
- return "si: "+dir.toString()+" "+name+" docCount: "+docCount+" delCount: "+delCount+" delFileName: "+getDelFileName();
- }
-
public SegmentInfo(String name, int docCount, Directory dir, Codec codec) {
this.name = name;
this.docCount = docCount;
@@ -730,29 +725,68 @@
sizeInBytes = -1;
}
- /** Used for debugging */
- public String segString(Directory dir) {
- String cfs;
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return toString(dir, 0);
+ }
+
+ /** Used for debugging. Format may suddenly change.
+ *
+ * <p>Current format looks like
+ * <code>_a:c45/4->_1</code>, which means the segment's
+ * name is <code>_a</code>; it's using compound file
+ * format (would be <code>C</code> if not compound); it
+ * has 45 documents; it has 4 deletions (this part is
+ * left off when there are no deletions); it's using the
+ * shared doc stores named <code>_1</code> (this part is
+ * left off if doc stores are private).</p>
+ */
+ public String toString(Directory dir, int pendingDelCount) {
+
+ StringBuilder s = new StringBuilder();
+ s.append(name).append(':');
+
+ char cfs;
try {
- if (getUseCompoundFile())
- cfs = "c";
- else
- cfs = "C";
+ if (getUseCompoundFile()) {
+ cfs = 'c';
+ } else {
+ cfs = 'C';
+ }
} catch (IOException ioe) {
- cfs = "?";
+ cfs = '?';
}
+ s.append(cfs);
- String docStore;
+ if (this.dir != dir) {
+ s.append('x');
+ }
+ s.append(docCount);
- if (docStoreOffset != -1)
- docStore = "->" + docStoreSegment;
- else
- docStore = "";
+ int delCount;
+ try {
+ delCount = getDelCount();
+ } catch (IOException ioe) {
+ delCount = -1;
+ }
+ if (delCount != -1) {
+ delCount += pendingDelCount;
+ }
+ if (delCount != 0) {
+ s.append('/');
+ if (delCount == -1) {
+ s.append('?');
+ } else {
+ s.append(delCount);
+ }
+ }
+
+ if (docStoreOffset != -1) {
+ s.append("->").append(docStoreSegment);
+ }
- return name + ":" +
- cfs +
- (this.dir == dir ? "" : "x") +
- docCount + docStore;
+ return s.toString();
}
/** We consider another SegmentInfo instance equal if it
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java Thu Jan 14 19:05:12 2010
@@ -878,17 +878,16 @@
finishCommit(dir);
}
- public synchronized String segString(Directory directory) {
+ public synchronized String toString(Directory directory) {
StringBuilder buffer = new StringBuilder();
+ buffer.append(getCurrentSegmentFileName()).append(": ");
final int count = size();
for(int i = 0; i < count; i++) {
if (i > 0) {
buffer.append(' ');
}
final SegmentInfo info = info(i);
- buffer.append(info.segString(directory));
- if (info.dir != directory)
- buffer.append("**");
+ buffer.append(info.toString(directory, 0));
}
return buffer.toString();
}
@@ -923,4 +922,14 @@
return true;
return false;
}
+
+ /** Returns sum of all segment's docCounts. Note that
+ * this does not include deletions */
+ public int totalDocCount() {
+ int count = 0;
+ for(SegmentInfo info : this) {
+ count += info.docCount;
+ }
+ return count;
+ }
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Thu Jan 14 19:05:12 2010
@@ -46,6 +46,7 @@
import org.apache.lucene.index.codecs.preflex.SegmentTermDocs;
import org.apache.lucene.index.codecs.preflex.SegmentTermPositions;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
/** @version $Id */
/**
@@ -103,12 +104,14 @@
final int readBufferSize;
final int termsIndexDivisor;
+ private final SegmentReader origInstance;
+
FieldsReader fieldsReaderOrig;
TermVectorsReader termVectorsReaderOrig;
CompoundFileReader cfsReader;
CompoundFileReader storeCFSReader;
- CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException {
+ CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException {
if (termsIndexDivisor < 1 && termsIndexDivisor != -1) {
throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + termsIndexDivisor);
@@ -153,6 +156,12 @@
decRef();
}
}
+
+ // Must assign this at the end -- if we hit an
+ // exception above core, we don't want to attempt to
+ // purge the FieldCache (will hit NPE because core is
+ // not assigned yet).
+ this.origInstance = origInstance;
}
synchronized TermVectorsReader getTermVectorsReaderOrig() {
@@ -194,6 +203,11 @@
if (storeCFSReader != null) {
storeCFSReader.close();
}
+
+ // Force FieldCache to evict our entries at this point
+ if (origInstance != null) {
+ FieldCache.DEFAULT.purge(origInstance);
+ }
}
}
@@ -515,7 +529,7 @@
boolean success = false;
try {
- instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor, codecs);
+ instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor, codecs);
if (doOpenStores) {
instance.core.openDocStores(si);
}
@@ -545,20 +559,31 @@
return deletedDocs;
}
+ private boolean checkDeletedCounts() throws IOException {
+ final int recomputedCount = deletedDocs.getRecomputedCount();
+
+ assert deletedDocs.count() == recomputedCount : "deleted count=" + deletedDocs.count() + " vs recomputed count=" + recomputedCount;
+
+ assert si.getDelCount() == recomputedCount :
+ "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + recomputedCount;
+
+ // Verify # deletes does not exceed maxDoc for this
+ // segment:
+ assert si.getDelCount() <= maxDoc() :
+ "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name;
+
+ return true;
+ }
+
private void loadDeletedDocs() throws IOException {
// NOTE: the bitvector is stored using the regular directory, not cfs
if (hasDeletions(si)) {
deletedDocs = new BitVector(directory(), si.getDelFileName());
deletedDocsRef = new AtomicInteger(1);
-
- assert si.getDelCount() == deletedDocs.count() :
- "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
-
- // Verify # deletes does not exceed maxDoc for this
- // segment:
- assert si.getDelCount() <= maxDoc() :
- "delete count mismatch: " + deletedDocs.count() + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name;
-
+ assert checkDeletedCounts();
+ if (deletedDocs.size() != si.docCount) {
+ throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount);
+ }
} else
assert si.getDelCount() == 0;
}
@@ -631,10 +656,10 @@
clone.readOnly = openReadOnly;
clone.si = si;
clone.readBufferSize = readBufferSize;
+ clone.pendingDeleteCount = pendingDeleteCount;
if (!openReadOnly && hasChanges) {
// My pending changes transfer to the new reader
- clone.pendingDeleteCount = pendingDeleteCount;
clone.deletedDocsDirty = deletedDocsDirty;
clone.normsDirty = normsDirty;
clone.hasChanges = hasChanges;
@@ -1206,6 +1231,17 @@
return termVectorsReader.get(docNumber);
}
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder();
+ if (hasChanges) {
+ buffer.append('*');
+ }
+ buffer.append(si.toString(core.dir, pendingDeleteCount));
+ return buffer.toString();
+ }
+
/**
* Return the name of the segment this reader is reading.
*/
@@ -1276,6 +1312,7 @@
* We do it with R/W access for the tests (BW compatibility)
* @deprecated Remove this when tests are fixed!
*/
+ @Deprecated
static SegmentReader getOnlySegmentReader(Directory dir) throws IOException {
return getOnlySegmentReader(IndexReader.open(dir, false));
}
@@ -1295,6 +1332,7 @@
throw new IllegalArgumentException(reader + " is not a SegmentReader or a single-segment DirectoryReader");
}
+ @Override
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java Thu Jan 14 19:05:12 2010
@@ -96,6 +96,7 @@
this.fields = fields;
}
+ @Override
protected Query getFieldQuery(String field, String queryText, int slop) throws ParseException {
if (field == null) {
List<BooleanClause> clauses = new ArrayList<BooleanClause>();
@@ -132,11 +133,13 @@
}
+ @Override
protected Query getFieldQuery(String field, String queryText) throws ParseException {
return getFieldQuery(field, queryText, 0);
}
+ @Override
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException
{
if (field == null) {
@@ -150,6 +153,7 @@
return super.getFuzzyQuery(field, termStr, minSimilarity);
}
+ @Override
protected Query getPrefixQuery(String field, String termStr) throws ParseException
{
if (field == null) {
@@ -163,6 +167,7 @@
return super.getPrefixQuery(field, termStr);
}
+ @Override
protected Query getWildcardQuery(String field, String termStr) throws ParseException {
if (field == null) {
List<BooleanClause> clauses = new ArrayList<BooleanClause>();
@@ -176,6 +181,7 @@
}
+ @Override
protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException {
if (field == null) {
List<BooleanClause> clauses = new ArrayList<BooleanClause>();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java Thu Jan 14 19:05:12 2010
@@ -614,6 +614,15 @@
public abstract void purgeAllCaches();
/**
+ * Expert: drops all cache entries associated with this
+ * reader. NOTE: this reader must precisely match the
+ * reader that the cache entry is keyed on. If you pass a
+ * top-level reader, it usually will have no effect as
+ * Lucene now caches at the segment reader level.
+ */
+ public abstract void purge(IndexReader r);
+
+ /**
* If non-null, FieldCacheImpl will warn whenever
* entries are created that are not sane according to
* {@link org.apache.lucene.util.FieldCacheSanityChecker}.
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Thu Jan 14 19:05:12 2010
@@ -65,6 +65,12 @@
public void purgeAllCaches() {
init();
}
+
+ public void purge(IndexReader r) {
+ for(Cache c : caches.values()) {
+ c.purge(r);
+ }
+ }
public CacheEntry[] getCacheEntries() {
List<CacheEntry> result = new ArrayList<CacheEntry>(17);
@@ -148,6 +154,14 @@
protected abstract Object createValue(IndexReader reader, Entry key)
throws IOException;
+ /** Remove this reader from the cache, if present. */
+ public void purge(IndexReader r) {
+ Object readerKey = r.getFieldCacheKey();
+ synchronized(readerCache) {
+ readerCache.remove(readerKey);
+ }
+ }
+
public Object get(IndexReader reader, Entry key) throws IOException {
Map<Entry,Object> innerCache;
Object value;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SortField.java Thu Jan 14 19:05:12 2010
@@ -21,10 +21,6 @@
import java.io.Serializable;
import java.util.Locale;
-import org.apache.lucene.document.NumericField; // javadocs
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
import org.apache.lucene.util.StringHelper;
/**
@@ -258,6 +254,13 @@
return reverse;
}
+ /** Returns the {@link FieldComparatorSource} used for
+ * custom sorting
+ */
+ public FieldComparatorSource getComparatorSource() {
+ return comparatorSource;
+ }
+
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Thu Jan 14 19:05:12 2010
@@ -207,6 +207,9 @@
* <pre>
* ModifiedScore = subQueryScore * valSrcScore
* </pre>
+ *
+ * <p><b>NOTE</b>: The doc is relative to the current
+ * reader, last passed to {@link #setNextReader}.
*
* @param doc id of scored doc.
* @param subQueryScore score of that doc by the subQuery.
@@ -218,6 +221,15 @@
}
/**
+ * Called when the scoring switches to another reader.
+ *
+ * @param reader
+ * next IndexReader
+ */
+ public void setNextReader(IndexReader reader) throws IOException {
+ }
+
+ /**
* Explain the custom score.
* Whenever overriding {@link #customScore(int, float, float[])},
* this method should also be overridden to provide the correct explanation
@@ -385,7 +397,6 @@
* A scorer that applies a (callback) function on scores of the subQuery.
*/
private class CustomScorer extends Scorer {
- private final CustomWeight weight;
private final float qWeight;
private Scorer subQueryScorer;
private Scorer[] valSrcScorers;
@@ -396,12 +407,12 @@
private CustomScorer(Similarity similarity, IndexReader reader, CustomWeight w,
Scorer subQueryScorer, Scorer[] valSrcScorers) throws IOException {
super(similarity);
- this.weight = w;
this.qWeight = w.getValue();
this.subQueryScorer = subQueryScorer;
this.valSrcScorers = valSrcScorers;
this.reader = reader;
this.vScores = new float[valSrcScorers.length];
+ setNextReader(reader);
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Thu Jan 14 19:05:12 2010
@@ -59,7 +59,7 @@
* <pre>
* SpanQuery q1 = new SpanTermQuery(new Term("studentfirstname", "james"));
* SpanQuery q2 = new SpanTermQuery(new Term("studentsurname", "jones"));
- * SpanQuery q2m new FieldMaskingSpanQuery(q2, "studentfirstname");
+ * SpanQuery q2m = new FieldMaskingSpanQuery(q2, "studentfirstname");
* Query q = new SpanNearQuery(new SpanQuery[]{q1, q2m}, -1, false);
* </pre>
* to search for 'studentfirstname:james studentsurname:jones' and find
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/FSDirectory.java Thu Jan 14 19:05:12 2010
@@ -373,6 +373,7 @@
}
/** @deprecated Use {@link #getDirectory} instead. */
+ @Deprecated
public File getFile() {
return getDirectory();
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java Thu Jan 14 19:05:12 2010
@@ -146,6 +146,7 @@
* instead, and construct the string
* from those utf8 bytes
*/
+ @Deprecated
public void readChars(char[] buffer, int start, int length)
throws IOException {
final int end = start + length;
@@ -174,6 +175,7 @@
* @deprecated this method operates on old "modified utf8" encoded
* strings
*/
+ @Deprecated
public void skipChars(int length) throws IOException{
for (int i = 0; i < length; i++) {
byte b = readByte();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java Thu Jan 14 19:05:12 2010
@@ -114,6 +114,7 @@
* @deprecated -- please pre-convert to utf8 bytes
* instead or use {@link #writeString}
*/
+ @Deprecated
public void writeChars(String s, int start, int length)
throws IOException {
final int end = start + length;
@@ -139,6 +140,7 @@
* @param length the number of characters in the sequence
* @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
*/
+ @Deprecated
public void writeChars(char[] s, int start, int length)
throws IOException {
final int end = start + length;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/MMapDirectory.java Thu Jan 14 19:05:12 2010
@@ -401,7 +401,7 @@
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
ensureOpen();
- File f = new File(getFile(), name);
+ File f = new File(getDirectory(), name);
RandomAccessFile raf = new RandomAccessFile(f, "r");
try {
return (raf.length() <= (long) maxBBuf)
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=899359&r1=899358&r2=899359&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/NIOFSDirectory.java Thu Jan 14 19:05:12 2010
@@ -64,7 +64,7 @@
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
ensureOpen();
- return new NIOFSIndexInput(new File(getFile(), name), bufferSize, getReadChunkSize());
+ return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
}
/** Creates an IndexOutput for the file with the given name. */