You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/12/10 17:37:54 UTC
svn commit: r1419570 [2/14] - in /lucene/dev/branches/lucene4547: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/
dev-tools/idea/solr/contrib/dataimporthandler/ dev-tools/maven/
dev-tools/maven/lucen...
Modified: lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java Mon Dec 10 16:36:47 2012
@@ -25,32 +25,68 @@ import java.io.IOException;
/**
* This TokenFilter limits the number of tokens while indexing. It is
* a replacement for the maximum field length setting inside {@link org.apache.lucene.index.IndexWriter}.
+ * <p>
+ * By default, this filter ignores any tokens in the wrapped {@code TokenStream}
+ * once the limit has been reached, which can result in {@code reset()} being
+ * called prior to {@code incrementToken()} returning {@code false}. For most
+ * {@code TokenStream} implementations this should be acceptable, and faster
+ * then consuming the full stream. If you are wrapping a {@code TokenStream}
+ * which requires that the full stream of tokens be exhausted in order to
+ * function properly, use the
+ * {@link #LimitTokenCountFilter(TokenStream,int,boolean) consumeAllTokens}
+ * option.
*/
public final class LimitTokenCountFilter extends TokenFilter {
private final int maxTokenCount;
+ private final boolean consumeAllTokens;
private int tokenCount = 0;
+ private boolean exhausted = false;
/**
* Build a filter that only accepts tokens up to a maximum number.
+ * This filter will not consume any tokens beyond the maxTokenCount limit
+ *
+ * @see #LimitTokenCountFilter(TokenStream,int,boolean)
*/
public LimitTokenCountFilter(TokenStream in, int maxTokenCount) {
+ this(in, maxTokenCount, false);
+ }
+
+ /**
+ * Build an filter that limits the maximum number of tokens per field.
+ * @param in the stream to wrap
+ * @param maxTokenCount max number of tokens to produce
+ * @param consumeAllTokens whether all tokens from the input must be consumed even if maxTokenCount is reached.
+ */
+ public LimitTokenCountFilter(TokenStream in, int maxTokenCount, boolean consumeAllTokens) {
super(in);
this.maxTokenCount = maxTokenCount;
+ this.consumeAllTokens = consumeAllTokens;
}
@Override
public boolean incrementToken() throws IOException {
- if (tokenCount < maxTokenCount && input.incrementToken()) {
- tokenCount++;
- return true;
+ if (exhausted) {
+ return false;
+ } else if (tokenCount < maxTokenCount) {
+ if (input.incrementToken()) {
+ tokenCount++;
+ return true;
+ } else {
+ exhausted = true;
+ return false;
+ }
+ } else {
+ while (consumeAllTokens && input.incrementToken()) { /* NOOP */ }
+ return false;
}
- return false;
}
@Override
public void reset() throws IOException {
super.reset();
tokenCount = 0;
+ exhausted = false;
}
}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java Mon Dec 10 16:36:47 2012
@@ -29,25 +29,29 @@ import org.apache.lucene.analysis.util.T
* <fieldType name="text_lngthcnt" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10"/>
+ * <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10" consumeAllTokens="false" />
* </analyzer>
* </fieldType></pre>
- *
+ * <p>
+ * The {@code consumeAllTokens} property is optional and defaults to {@code false}. See {@link LimitTokenCountFilter} for an explanation of it's use.
*/
public class LimitTokenCountFilterFactory extends TokenFilterFactory {
public static final String MAX_TOKEN_COUNT_KEY = "maxTokenCount";
+ public static final String CONSUME_ALL_TOKENS_KEY = "consumeAllTokens";
int maxTokenCount;
+ boolean consumeAllTokens;
@Override
public void init(Map<String, String> args) {
super.init( args );
maxTokenCount = getInt(MAX_TOKEN_COUNT_KEY);
+ consumeAllTokens = getBoolean(CONSUME_ALL_TOKENS_KEY, false);
}
@Override
public TokenStream create(TokenStream input) {
- return new LimitTokenCountFilter(input, maxTokenCount);
+ return new LimitTokenCountFilter(input, maxTokenCount, consumeAllTokens);
}
}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Mon Dec 10 16:36:47 2012
@@ -59,6 +59,7 @@ import org.apache.lucene.analysis.Valida
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
import org.apache.lucene.analysis.cjk.CJKBigramFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
+import org.apache.lucene.analysis.commongrams.CommonGramsQueryFilter;
import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter;
@@ -160,7 +161,9 @@ public class TestRandomChains extends Ba
// TODO: doesn't handle graph inputs (or even look at positionIncrement)
HyphenatedWordsFilter.class,
// LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- TypeTokenFilter.class
+ TypeTokenFilter.class,
+ // TODO: doesn't handle graph inputs
+ CommonGramsQueryFilter.class
);
}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Mon Dec 10 16:36:47 2012
@@ -33,40 +33,64 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util._TestUtil;
public class TestLimitTokenCountAnalyzer extends BaseTokenStreamTestCase {
public void testLimitTokenCountAnalyzer() throws IOException {
- Analyzer a = new LimitTokenCountAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, 4);
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
+ for (boolean consumeAll : new boolean[] { true, false }) {
+ MockAnalyzer mock = new MockAnalyzer(random());
+
+ // if we are consuming all tokens, we can use the checks,
+ // otherwise we can't
+ mock.setEnableChecks(consumeAll);
+ Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
+
+ // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
+
+ // less than the limit, ensure we behave correctly
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
- a = new LimitTokenCountAnalyzer(new StandardAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
+ // equal to limit
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
+ }
}
public void testLimitTokenCountIndexWriter() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new LimitTokenCountAnalyzer(new MockAnalyzer(random()), 100000)));
-
- Document doc = new Document();
- StringBuilder b = new StringBuilder();
- for(int i=0;i<10000;i++)
- b.append(" a");
- b.append(" x");
- doc.add(newTextField("field", b.toString(), Field.Store.NO));
- writer.addDocument(doc);
- writer.close();
-
- IndexReader reader = DirectoryReader.open(dir);
- Term t = new Term("field", "x");
- assertEquals(1, reader.docFreq(t));
- reader.close();
- dir.close();
+
+ for (boolean consumeAll : new boolean[] { true, false }) {
+ Directory dir = newDirectory();
+ int limit = _TestUtil.nextInt(random(), 50, 101000);
+ MockAnalyzer mock = new MockAnalyzer(random());
+
+ // if we are consuming all tokens, we can use the checks,
+ // otherwise we can't
+ mock.setEnableChecks(consumeAll);
+ Analyzer a = new LimitTokenCountAnalyzer(mock, limit, consumeAll);
+
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig
+ (TEST_VERSION_CURRENT, a));
+
+ Document doc = new Document();
+ StringBuilder b = new StringBuilder();
+ for(int i=1;i<limit;i++)
+ b.append(" a");
+ b.append(" x");
+ b.append(" z");
+ doc.add(newTextField("field", b.toString(), Field.Store.NO));
+ writer.addDocument(doc);
+ writer.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+ Term t = new Term("field", "x");
+ assertEquals(1, reader.docFreq(t));
+ t = new Term("field", "z");
+ assertEquals(0, reader.docFreq(t));
+ reader.close();
+ dir.close();
+ }
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/icu/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/icu/build.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/icu/build.xml (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/icu/build.xml Mon Dec 10 16:36:47 2012
@@ -35,6 +35,11 @@
<path refid="base.classpath"/>
</path>
+ <path id="test.classpath">
+ <path refid="test.base.classpath" />
+ <pathelement path="src/test-files" />
+ </path>
+
<target name="compile-core" depends="jar-analyzers-common, common.compile-core" />
<property name="utr30.data.dir" location="src/data/utr30"/>
Modified: lucene/dev/branches/lucene4547/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java Mon Dec 10 16:36:47 2012
@@ -17,22 +17,135 @@ package org.apache.lucene.analysis.icu.s
* limitations under the License.
*/
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Reader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.IOUtils;
-/** Factory for {@link ICUTokenizer} */
-public class ICUTokenizerFactory extends TokenizerFactory {
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.lang.UProperty;
+import com.ibm.icu.lang.UScript;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+
+/**
+ * Factory for {@link ICUTokenizer}.
+ * Words are broken across script boundaries, then segmented according to
+ * the BreakIterator and typing provided by the {@link DefaultICUTokenizerConfig}.
+ *
+ * <p/>
+ *
+ * To use the default set of per-script rules:
+ *
+ * <pre class="prettyprint" >
+ * <fieldType name="text_icu" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.ICUTokenizerFactory"/>
+ * </analyzer>
+ * </fieldType></pre>
+ *
+ * <p/>
+ *
+ * You can customize this tokenizer's behavior by specifying per-script rule files,
+ * which are compiled by the ICU RuleBasedBreakIterator. See the
+ * <a href="http://userguide.icu-project.org/boundaryanalysis#TOC-RBBI-Rules"
+ * >ICU RuleBasedBreakIterator syntax reference</a>.
+ *
+ * To add per-script rules, add a "rulefiles" argument, which should contain a
+ * comma-separated list of <tt>code:rulefile</tt> pairs in the following format:
+ * <a href="http://unicode.org/iso15924/iso15924-codes.html"
+ * >four-letter ISO 15924 script code</a>, followed by a colon, then a resource
+ * path. E.g. to specify rules for Latin (script code "Latn") and Cyrillic
+ * (script code "Cyrl"):
+ *
+ * <pre class="prettyprint" >
+ * <fieldType name="text_icu_custom" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.ICUTokenizerFactory"
+ * rulefiles="Latn:my.Latin.rules.rbbi,Cyrl:my.Cyrillic.rules.rbbi"/>
+ * </analyzer>
+ * </fieldType></pre>
+ *
+ */
+public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware {
+ static final String RULEFILES = "rulefiles";
+ private Map<Integer,String> tailored;
+ private ICUTokenizerConfig config;
/** Sole constructor. See {@link AbstractAnalysisFactory} for initialization lifecycle. */
public ICUTokenizerFactory() {}
- // TODO: add support for custom configs
+ @Override
+ public void init(Map<String,String> args) {
+ super.init(args);
+ tailored = new HashMap<Integer,String>();
+ String rulefilesArg = args.get(RULEFILES);
+ if (rulefilesArg != null) {
+ List<String> scriptAndResourcePaths = splitFileNames(rulefilesArg);
+ for (String scriptAndResourcePath : scriptAndResourcePaths) {
+ int colonPos = scriptAndResourcePath.indexOf(":");
+ String scriptCode = scriptAndResourcePath.substring(0, colonPos).trim();
+ String resourcePath = scriptAndResourcePath.substring(colonPos+1).trim();
+ tailored.put(UCharacter.getPropertyValueEnum(UProperty.SCRIPT, scriptCode), resourcePath);
+ }
+ }
+ }
+
+ @Override
+ public void inform(ResourceLoader loader) throws IOException {
+ assert tailored != null : "init must be called first!";
+ if (tailored.isEmpty()) {
+ config = new DefaultICUTokenizerConfig();
+ } else {
+ final BreakIterator breakers[] = new BreakIterator[UScript.CODE_LIMIT];
+ for (Map.Entry<Integer,String> entry : tailored.entrySet()) {
+ int code = entry.getKey();
+ String resourcePath = entry.getValue();
+ breakers[code] = parseRules(resourcePath, loader);
+ }
+ config = new DefaultICUTokenizerConfig() {
+
+ @Override
+ public BreakIterator getBreakIterator(int script) {
+ if (breakers[script] != null) {
+ return (BreakIterator) breakers[script].clone();
+ } else {
+ return super.getBreakIterator(script);
+ }
+ }
+ // TODO: we could also allow codes->types mapping
+ };
+ }
+ }
+
+ private BreakIterator parseRules(String filename, ResourceLoader loader) throws IOException {
+ StringBuilder rules = new StringBuilder();
+ InputStream rulesStream = loader.openResource(filename);
+ BufferedReader reader = new BufferedReader
+ (IOUtils.getDecodingReader(rulesStream, IOUtils.CHARSET_UTF_8));
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ if ( ! line.startsWith("#"))
+ rules.append(line);
+ rules.append('\n');
+ }
+ reader.close();
+ return new RuleBasedBreakIterator(rules.toString());
+ }
+
@Override
public Tokenizer create(Reader input) {
- return new ICUTokenizer(input);
+ assert config != null : "inform must be called first!";
+ return new ICUTokenizer(input, config);
}
}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java Mon Dec 10 16:36:47 2012
@@ -19,18 +19,73 @@ package org.apache.lucene.analysis.icu.s
import java.io.Reader;
import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.ClasspathResourceLoader;
/** basic tests for {@link ICUTokenizerFactory} **/
public class TestICUTokenizerFactory extends BaseTokenStreamTestCase {
public void testMixedText() throws Exception {
Reader reader = new StringReader("à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ This is a test àºàº§à»àº²àºàºàº");
ICUTokenizerFactory factory = new ICUTokenizerFactory();
+ factory.init(new HashMap<String,String>());
+ factory.inform(new ClasspathResourceLoader(getClass()));
TokenStream stream = factory.create(reader);
assertTokenStreamContents(stream,
new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ",
"This", "is", "a", "test", "àºàº§à»àº²", "àºàºàº"});
}
+
+ public void testTokenizeLatinOnWhitespaceOnly() throws Exception {
+ // â U+201C LEFT DOUBLE QUOTATION MARK; â U+201D RIGHT DOUBLE QUOTATION MARK
+ Reader reader = new StringReader
+ (" Don't,break.at?/(punct)! \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$ ");
+ ICUTokenizerFactory factory = new ICUTokenizerFactory();
+ final Map<String,String> args = new HashMap<String,String>();
+ args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
+ factory.init(args);
+ factory.inform(new ClasspathResourceLoader(this.getClass()));
+ TokenStream stream = factory.create(reader);
+ assertTokenStreamContents(stream,
+ new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
+ new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<OTHER>" });
+ }
+
+ public void testTokenizeLatinDontBreakOnHyphens() throws Exception {
+ Reader reader = new StringReader
+ ("One-two punch. Brang-, not brung-it. This one--not that one--is the right one, -ish.");
+ ICUTokenizerFactory factory = new ICUTokenizerFactory();
+ final Map<String,String> args = new HashMap<String,String>();
+ args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
+ factory.init(args);
+ factory.inform(new ClasspathResourceLoader(getClass()));
+ TokenStream stream = factory.create(reader);
+ assertTokenStreamContents(stream,
+ new String[] { "One-two", "punch",
+ "Brang", "not", "brung-it",
+ "This", "one", "not", "that", "one", "is", "the", "right", "one", "ish" });
+ }
+
+ /**
+ * Specify more than one script/rule file pair.
+ * Override default DefaultICUTokenizerConfig Thai script tokenization.
+ * Use the same rule file for both scripts.
+ */
+ public void testKeywordTokenizeCyrillicAndThai() throws Exception {
+ Reader reader = new StringReader
+ ("Some English. Ðемного ÑÑÑÑкий. à¸à¹à¸à¸à¸§à¸²à¸¡à¸ าษาà¹à¸à¸¢à¹à¸¥à¹à¸ ๠à¸à¹à¸à¸¢ ๠More English.");
+ ICUTokenizerFactory factory = new ICUTokenizerFactory();
+ final Map<String,String> args = new HashMap<String,String>();
+ args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
+ factory.init(args);
+ factory.inform(new ClasspathResourceLoader(getClass()));
+ TokenStream stream = factory.create(reader);
+ assertTokenStreamContents(stream, new String[] { "Some", "English",
+ "Ðемного ÑÑÑÑкий. ",
+ "à¸à¹à¸à¸à¸§à¸²à¸¡à¸ าษาà¹à¸à¸¢à¹à¸¥à¹à¸ ๠à¸à¹à¸à¸¢ ๠",
+ "More", "English" });
+ }
}
Modified: lucene/dev/branches/lucene4547/lucene/analysis/uima/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/analysis/uima/build.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/analysis/uima/build.xml (original)
+++ lucene/dev/branches/lucene4547/lucene/analysis/uima/build.xml Mon Dec 10 16:36:47 2012
@@ -24,7 +24,7 @@
</description>
<property name="tests.userdir" value="src/test-files"/>
- <property name="tests.jvms" value="1" />
+ <property name="tests.jvms.override" value="1" />
<path id="uimajars">
<pathelement path="lib/uimaj-core-2.3.1.jar"/>
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/build.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/build.xml (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/build.xml Mon Dec 10 16:36:47 2012
@@ -26,9 +26,6 @@
<import file="../module-build.xml"/>
<property name="working.dir" location="work"/>
- <!-- the tests have some parallel problems -->
- <property name="tests.jvms" value="1"/>
-
<target name="check-files">
<available file="temp/news20.tar.gz" property="news20.exists"/>
@@ -155,7 +152,7 @@
<fileset dir="lib">
<include name="commons-compress-1.4.1.jar"/>
<include name="xercesImpl-2.9.1.jar"/>
- <include name="nekohtml-1.9.15.jar"/>
+ <include name="nekohtml-1.9.17.jar"/>
</fileset>
</path>
<path id="run.classpath">
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/conf/extractWikipedia.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/conf/extractWikipedia.alg?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/conf/extractWikipedia.alg (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/conf/extractWikipedia.alg Mon Dec 10 16:36:47 2012
@@ -41,4 +41,4 @@ content.source.forever=false
# -------------------------------------------------------------------------------------
# Process all documents, appending each one to the line file:
-{WriteLineDoc() > : *
+{WriteEnwikiLineDoc() > : *
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/ivy.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/ivy.xml (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/ivy.xml Mon Dec 10 16:36:47 2012
@@ -21,7 +21,7 @@
<dependencies>
<dependency org="org.apache.commons" name="commons-compress" rev="1.4.1" transitive="false"/>
<dependency org="xerces" name="xercesImpl" rev="2.9.1" transitive="false"/>
- <dependency org="net.sourceforge.nekohtml" name="nekohtml" rev="1.9.15" transitive="false"/>
+ <dependency org="net.sourceforge.nekohtml" name="nekohtml" rev="1.9.17" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java Mon Dec 10 16:36:47 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.benchmark.byTa
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
@@ -65,10 +66,10 @@ public class DemoHTMLParser implements H
@Override
public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
if (inHEAD > 0) {
- if (equalsIgnoreTurkish("title", localName)) {
+ if ("title".equals(localName)) {
inTITLE++;
} else {
- if (equalsIgnoreTurkish("meta", localName)) {
+ if ("meta".equals(localName)) {
String name = atts.getValue("name");
if (name == null) {
name = atts.getValue("http-equiv");
@@ -82,7 +83,7 @@ public class DemoHTMLParser implements H
} else if (inBODY > 0) {
if (SUPPRESS_ELEMENTS.contains(localName)) {
suppressed++;
- } else if (equalsIgnoreTurkish("img", localName)) {
+ } else if ("img".equals(localName)) {
// the original javacc-based parser preserved <IMG alt="..."/>
// attribute as body text in [] parenthesis:
final String alt = atts.getValue("alt");
@@ -90,11 +91,11 @@ public class DemoHTMLParser implements H
body.append('[').append(alt).append(']');
}
}
- } else if (equalsIgnoreTurkish("body", localName)) {
+ } else if ("body".equals(localName)) {
inBODY++;
- } else if (equalsIgnoreTurkish("head", localName)) {
+ } else if ("head".equals(localName)) {
inHEAD++;
- } else if (equalsIgnoreTurkish("frameset", localName)) {
+ } else if ("frameset".equals(localName)) {
throw new SAXException("This parser does not support HTML framesets.");
}
}
@@ -102,7 +103,7 @@ public class DemoHTMLParser implements H
@Override
public void endElement(String namespaceURI, String localName, String qName) throws SAXException {
if (inBODY > 0) {
- if (equalsIgnoreTurkish("body", localName)) {
+ if ("body".equals(localName)) {
inBODY--;
} else if (ENDLINE_ELEMENTS.contains(localName)) {
body.append('\n');
@@ -110,9 +111,9 @@ public class DemoHTMLParser implements H
suppressed--;
}
} else if (inHEAD > 0) {
- if (equalsIgnoreTurkish("head", localName)) {
+ if ("head".equals(localName)) {
inHEAD--;
- } else if (inTITLE > 0 && equalsIgnoreTurkish("title", localName)) {
+ } else if (inTITLE > 0 && "title".equals(localName)) {
inTITLE--;
}
}
@@ -145,36 +146,8 @@ public class DemoHTMLParser implements H
this.body = body.toString();
}
- // TODO: remove the Turkish workaround once this is fixed in NekoHTML:
- // https://sourceforge.net/tracker/?func=detail&aid=3544334&group_id=195122&atid=952178
-
- // BEGIN: workaround
- static final String convertTurkish(String s) {
- return s.replace('i', 'ı');
- }
-
- static final boolean equalsIgnoreTurkish(String s1, String s2) {
- final int len1 = s1.length(), len2 = s2.length();
- if (len1 != len2)
- return false;
- for (int i = 0; i < len1; i++) {
- char ch1 = s1.charAt(i), ch2 = s2.charAt(i);
- if (ch1 == 'ı') ch1 = 'i';
- if (ch2 == 'ı') ch2 = 'i';
- if (ch1 != ch2)
- return false;
- }
- return true;
- }
- // END: workaround
-
- static final Set<String> createElementNameSet(String... names) {
- final HashSet<String> set = new HashSet<String>();
- for (final String name : names) {
- set.add(name);
- set.add(convertTurkish(name));
- }
- return Collections.unmodifiableSet(set);
+ private static final Set<String> createElementNameSet(String... names) {
+ return Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(names)));
}
/** HTML elements that cause a line break (they are block-elements) */
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java Mon Dec 10 16:36:47 2012
@@ -53,6 +53,7 @@ public class EnwikiContentSource extends
private class Parser extends DefaultHandler implements Runnable {
private Thread t;
private boolean threadDone;
+ private boolean stopped = false;
private String[] tuple;
private NoMoreDataException nmde;
private StringBuilder contents = new StringBuilder();
@@ -70,31 +71,31 @@ public class EnwikiContentSource extends
}
String[] result;
synchronized(this){
- while(tuple == null && nmde == null && !threadDone) {
+ while(tuple == null && nmde == null && !threadDone && !stopped) {
try {
wait();
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
}
}
+ if (tuple != null) {
+ result = tuple;
+ tuple = null;
+ notify();
+ return result;
+ }
if (nmde != null) {
// Set to null so we will re-start thread in case
// we are re-used:
t = null;
throw nmde;
}
- if (t != null && threadDone) {
- // The thread has exited yet did not hit end of
- // data, so this means it hit an exception. We
- // throw NoMorDataException here to force
- // benchmark to stop the current alg:
- throw new NoMoreDataException();
- }
- result = tuple;
- tuple = null;
- notify();
+ // The thread has exited yet did not hit end of
+ // data, so this means it hit an exception. We
+ // throw NoMorDataException here to force
+ // benchmark to stop the current alg:
+ throw new NoMoreDataException();
}
- return result;
}
String time(String original) {
@@ -132,7 +133,7 @@ public class EnwikiContentSource extends
tmpTuple[BODY] = body.replaceAll("[\t\n]", " ");
tmpTuple[ID] = id;
synchronized(this) {
- while (tuple != null) {
+ while (tuple != null && !stopped) {
try {
wait();
} catch (InterruptedException ie) {
@@ -175,32 +176,33 @@ public class EnwikiContentSource extends
XMLReader reader = XMLReaderFactory.createXMLReader();
reader.setContentHandler(this);
reader.setErrorHandler(this);
- while(true){
+ while(!stopped){
final InputStream localFileIS = is;
- try {
- // To work around a bug in XERCES (XERCESJ-1257), we assume the XML is always UTF8, so we simply provide reader.
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
- .onMalformedInput(CodingErrorAction.REPORT)
- .onUnmappableCharacter(CodingErrorAction.REPORT);
- reader.parse(new InputSource(new BufferedReader(new InputStreamReader(localFileIS, decoder))));
- } catch (IOException ioe) {
- synchronized(EnwikiContentSource.this) {
- if (localFileIS != is) {
- // fileIS was closed on us, so, just fall
- // through
- } else
- // Exception is real
- throw ioe;
+ if (localFileIS != null) { // null means fileIS was closed on us
+ try {
+ // To work around a bug in XERCES (XERCESJ-1257), we assume the XML is always UTF8, so we simply provide reader.
+ CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT);
+ reader.parse(new InputSource(new BufferedReader(new InputStreamReader(localFileIS, decoder))));
+ } catch (IOException ioe) {
+ synchronized(EnwikiContentSource.this) {
+ if (localFileIS != is) {
+ // fileIS was closed on us, so, just fall through
+ } else
+ // Exception is real
+ throw ioe;
+ }
}
}
synchronized(this) {
- if (!forever) {
+ if (stopped || !forever) {
nmde = new NoMoreDataException();
notify();
return;
} else if (localFileIS == is) {
// If file is not already re-opened then re-open it now
- is = StreamUtils.inputStream(file);
+ is = openInputStream();
}
}
}
@@ -238,6 +240,17 @@ public class EnwikiContentSource extends
// this element should be discarded.
}
}
+
+ private void stop() {
+ synchronized (this) {
+ stopped = true;
+ if (tuple != null) {
+ tuple = null;
+ notify();
+ }
+ }
+ }
+
}
private static final Map<String,Integer> ELEMENTS = new HashMap<String,Integer>();
@@ -280,6 +293,7 @@ public class EnwikiContentSource extends
@Override
public void close() throws IOException {
synchronized (EnwikiContentSource.this) {
+ parser.stop();
if (is != null) {
is.close();
is = null;
@@ -301,7 +315,12 @@ public class EnwikiContentSource extends
@Override
public void resetInputs() throws IOException {
super.resetInputs();
- is = StreamUtils.inputStream(file);
+ is = openInputStream();
+ }
+
+ /** Open the input stream. */
+ protected InputStream openInputStream() throws IOException {
+ return StreamUtils.inputStream(file);
}
@Override
@@ -309,10 +328,9 @@ public class EnwikiContentSource extends
super.setConfig(config);
keepImages = config.get("keep.image.only.docs", true);
String fileName = config.get("docs.file", null);
- if (fileName == null) {
- throw new IllegalArgumentException("docs.file must be set");
+ if (fileName != null) {
+ file = new File(fileName).getAbsoluteFile();
}
- file = new File(fileName).getAbsoluteFile();
}
}
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java Mon Dec 10 16:36:47 2012
@@ -49,7 +49,8 @@ public class CommitIndexTask extends Per
public int doLogic() throws Exception {
IndexWriter iw = getRunData().getIndexWriter();
if (iw != null) {
- iw.commit(commitUserData);
+ iw.setCommitData(commitUserData);
+ iw.commit();
}
return 1;
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java Mon Dec 10 16:36:47 2012
@@ -32,7 +32,6 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
/**
@@ -84,18 +83,20 @@ public class WriteLineDocTask extends Pe
public static final String DEFAULT_SUFFICIENT_FIELDS = DocMaker.TITLE_FIELD +',' + DocMaker.BODY_FIELD;
private int docSize = 0;
- private PrintWriter lineFileOut = null;
- private DocMaker docMaker;
- private ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
- private ThreadLocal<Matcher> threadNormalizer = new ThreadLocal<Matcher>();
- private final String[] fieldsToWrite;;
+ protected final String fname;
+ private final PrintWriter lineFileOut;
+ private final DocMaker docMaker;
+ private final ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
+ private final ThreadLocal<Matcher> threadNormalizer = new ThreadLocal<Matcher>();
+ private final String[] fieldsToWrite;
private final boolean[] sufficientFields;
private final boolean checkSufficientFields;
+
public WriteLineDocTask(PerfRunData runData) throws Exception {
super(runData);
Config config = runData.getConfig();
- String fname = config.get("line.file.out", null);
+ fname = config.get("line.file.out", null);
if (fname == null) {
throw new IllegalArgumentException("line.file.out must be set");
}
@@ -129,13 +130,13 @@ public class WriteLineDocTask extends Pe
}
}
- writeHeader();
+ writeHeader(lineFileOut);
}
/**
- * Write a header to the lines file - indicating how to read the file later
+ * Write header to the lines file - indicating how to read the file later.
*/
- private void writeHeader() {
+ protected void writeHeader(PrintWriter out) {
StringBuilder sb = threadBuffer.get();
if (sb == null) {
sb = new StringBuilder();
@@ -146,7 +147,7 @@ public class WriteLineDocTask extends Pe
for (String f : fieldsToWrite) {
sb.append(SEP).append(f);
}
- lineFileOut.println(sb.toString());
+ out.println(sb.toString());
}
@Override
@@ -181,12 +182,20 @@ public class WriteLineDocTask extends Pe
if (sufficient) {
sb.setLength(sb.length()-1); // remove redundant last separator
// lineFileOut is a PrintWriter, which synchronizes internally in println.
- lineFileOut.println(sb.toString());
+ lineFileOut(doc).println(sb.toString());
}
return 1;
}
+ /**
+ * Selects output line file by written doc.
+ * Default: original output line file.
+ */
+ protected PrintWriter lineFileOut(Document doc) {
+ return lineFileOut;
+ }
+
@Override
public void close() throws Exception {
lineFileOut.close();
Modified: lucene/dev/branches/lucene4547/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (original)
+++ lucene/dev/branches/lucene4547/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java Mon Dec 10 16:36:47 2012
@@ -188,7 +188,7 @@ public class WriteLineDocTaskTest extend
}
}
- private void assertHeaderLine(String line) {
+ static void assertHeaderLine(String line) {
assertTrue("First line should be a header line",line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR));
}
Modified: lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java (original)
+++ lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java Mon Dec 10 16:36:47 2012
@@ -52,14 +52,20 @@ public class KNearestNeighborClassifier
this.k = k;
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public ClassificationResult assignClass(String text) throws IOException {
Query q = mlt.like(new StringReader(text), textFieldName);
- TopDocs docs = indexSearcher.search(q, k);
+ TopDocs topDocs = indexSearcher.search(q, k);
+ return selectClassFromNeighbors(topDocs);
+ }
+ private ClassificationResult selectClassFromNeighbors(TopDocs topDocs) throws IOException {
// TODO : improve the nearest neighbor selection
Map<String, Integer> classCounts = new HashMap<String, Integer>();
- for (ScoreDoc scoreDoc : docs.scoreDocs) {
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
String cl = indexSearcher.doc(scoreDoc.doc).getField(classFieldName).stringValue();
Integer count = classCounts.get(cl);
if (count != null) {
@@ -68,7 +74,7 @@ public class KNearestNeighborClassifier
classCounts.put(cl, 1);
}
}
- int max = 0;
+ double max = 0;
String assignedClass = null;
for (String cl : classCounts.keySet()) {
Integer count = classCounts.get(cl);
@@ -77,10 +83,13 @@ public class KNearestNeighborClassifier
assignedClass = cl;
}
}
- double score = max / k;
+ double score = max / (double) k;
return new ClassificationResult(assignedClass, score);
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public void train(AtomicReader atomicReader, String textFieldName, String classFieldName, Analyzer analyzer) throws IOException {
this.textFieldName = textFieldName;
Modified: lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java (original)
+++ lucene/dev/branches/lucene4547/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java Mon Dec 10 16:36:47 2012
@@ -38,6 +38,7 @@ import java.util.LinkedList;
/**
* A simplistic Lucene based NaiveBayes classifier, see <code>http://en.wikipedia.org/wiki/Naive_Bayes_classifier</code>
+ *
* @lucene.experimental
*/
public class SimpleNaiveBayesClassifier implements Classifier {
@@ -48,14 +49,19 @@ public class SimpleNaiveBayesClassifier
private int docsWithClassSize;
private Analyzer analyzer;
private IndexSearcher indexSearcher;
-
- /**
+
+ /**
* Creates a new NaiveBayes classifier.
* Note that you must call {@link #train(AtomicReader, String, String, Analyzer) train()} before you can
* classify any documents.
*/
- public SimpleNaiveBayesClassifier() {}
+ public SimpleNaiveBayesClassifier() {
+ }
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void train(AtomicReader atomicReader, String textFieldName, String classFieldName, Analyzer analyzer)
throws IOException {
this.atomicReader = atomicReader;
@@ -79,6 +85,10 @@ public class SimpleNaiveBayesClassifier
return result.toArray(new String[result.size()]);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public ClassificationResult assignClass(String inputDocument) throws IOException {
if (atomicReader == null) {
throw new RuntimeException("need to train the classifier first");
@@ -89,9 +99,10 @@ public class SimpleNaiveBayesClassifier
Terms terms = MultiFields.getTerms(atomicReader, classFieldName);
TermsEnum termsEnum = terms.iterator(null);
BytesRef next;
- while((next = termsEnum.next()) != null) {
+ String[] tokenizedDoc = tokenizeDoc(inputDocument);
+ while ((next = termsEnum.next()) != null) {
// TODO : turn it to be in log scale
- double clVal = calculatePrior(next) * calculateLikelihood(inputDocument, next);
+ double clVal = calculatePrior(next) * calculateLikelihood(tokenizedDoc, next);
if (clVal > max) {
max = clVal;
foundClass = next.utf8ToString();
@@ -101,10 +112,10 @@ public class SimpleNaiveBayesClassifier
}
- private double calculateLikelihood(String document, BytesRef c) throws IOException {
+ private double calculateLikelihood(String[] tokenizedDoc, BytesRef c) throws IOException {
// for each word
double result = 1d;
- for (String word : tokenizeDoc(document)) {
+ for (String word : tokenizedDoc) {
// search with text:word AND class:c
int hits = getWordFreqForClass(word, c);
Modified: lucene/dev/branches/lucene4547/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java (original)
+++ lucene/dev/branches/lucene4547/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java Mon Dec 10 16:36:47 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.classification
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
@@ -71,52 +72,57 @@ public abstract class ClassificationTest
private void populateIndex(Analyzer analyzer) throws Exception {
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.setStoreTermVectorOffsets(true);
+ ft.setStoreTermVectorPositions(true);
+
Document doc = new Document();
- doc.add(new TextField(textFieldName, "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " +
+ doc.add(new Field(textFieldName, "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " +
"who attempted to ask questions of the Republican presidential candidate in a public plaza near the Tomb of " +
- "the Unknown Soldier in Warsaw Tuesday.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "the Unknown Soldier in Warsaw Tuesday.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" +
- " States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ doc.add(new Field(textFieldName, "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" +
+ " States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "And there's a threshold question that he has to answer for the American people and " +
+ doc.add(new Field(textFieldName, "And there's a threshold question that he has to answer for the American people and " +
"that's whether he is prepared to be commander-in-chief,\" she continued. \"As we look to the past events, we " +
- "know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\"", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\"", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Still, when it comes to gun policy, many congressional Democrats have \"decided to " +
+ doc.add(new Field(textFieldName, "Still, when it comes to gun policy, many congressional Democrats have \"decided to " +
"keep quiet and not go there,\" said Alan Lizotte, dean and professor at the State University of New York at " +
- "Albany's School of Criminal Justice.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "Albany's School of Criminal Justice.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " +
+ doc.add(new Field(textFieldName, "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " +
"technology at the University of Wisconsin-La Crosse, documented the historic moment and shared it with the " +
- "world through the Internet.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ "world through the Internet.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "So, about all those experts and analysts who've spent the past year or so saying " +
- "Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ doc.add(new Field(textFieldName, "So, about all those experts and analysts who've spent the past year or so saying " +
+ "Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "More than 400 million people trust Google with their e-mail, and 50 million store files" +
+ doc.add(new Field(textFieldName, "More than 400 million people trust Google with their e-mail, and 50 million store files" +
" in the cloud using the Dropbox service. People manage their bank accounts, pay bills, trade stocks and " +
- "generally transfer or store huge volumes of personal data online.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ "generally transfer or store huge volumes of personal data online.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
indexWriter.commit();
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Mon Dec 10 16:36:47 2012
@@ -97,12 +97,6 @@ public class SimpleTextTermVectorsReader
@Override
public Fields get(int doc) throws IOException {
- // TestTV tests for this in testBadParams... but is this
- // really guaranteed by the API?
- if (doc < 0 || doc >= offsets.length) {
- throw new IllegalArgumentException("doc id out of range");
- }
-
SortedMap<String,SimpleTVTerms> fields = new TreeMap<String,SimpleTVTerms>();
in.seek(offsets[doc]);
readLine();
Modified: lucene/dev/branches/lucene4547/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/common-build.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/common-build.xml (original)
+++ lucene/dev/branches/lucene4547/lucene/common-build.xml Mon Dec 10 16:36:47 2012
@@ -59,8 +59,6 @@
<property name="common.classpath.excludes" value="**/*.txt,**/*.template,**/*.sha1" />
<property name="ivy.bootstrap.version" value="2.2.0" />
- <property name="ivy.resource" value="org/apache/ivy/ant/antlib.xml" />
- <available resource="${ivy.resource}" property="ivy.available" />
<property name="ivy.default.configuration" value="*"/>
<property name="ivy.sync" value="true"/>
@@ -82,7 +80,11 @@
<property name="args" value=""/>
<property name="tests.seed" value="" />
+
+ <!-- This is a hack to be able to override the JVM count for special modules that don't like parallel tests: -->
<property name="tests.jvms" value="auto" />
+ <property name="tests.jvms.override" value="${tests.jvms}" />
+
<property name="tests.multiplier" value="1" />
<property name="tests.codec" value="random" />
<property name="tests.postingsformat" value="random" />
@@ -309,7 +311,7 @@
<!-- <property name="ivy.settings.uptodate" value="true"/> -->
</target>
- <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="resolve" depends="ivy-availability-check,ivy-configure">
<!-- todo, make this a property or something.
only special cases need bundles -->
<ivy:retrieve type="jar,bundle" log="download-only"
@@ -323,6 +325,13 @@
<property name="ivy_checksum_sha1" value="f9d1e83e82fc085093510f7d2e77d81d52bc2081"/>
<target name="ivy-availability-check" unless="ivy.available">
+ <condition property="ivy.available">
+ <typefound uri="antlib:org.apache.ivy.ant" name="configure" />
+ </condition>
+ <antcall target="ivy-fail" />
+ </target>
+
+ <target name="ivy-fail" unless="ivy.available">
<echo>
This build requires Ivy and Ivy could not be found in your ant classpath.
@@ -356,10 +365,9 @@
ant config to allow the user lib to be loaded. See the wiki for more details:
http://wiki.apache.org/lucene-java/HowToContribute#antivy
</echo>
- </target>
- <target name="ivy-fail" unless="ivy.available">
<fail>Ivy is not available</fail>
</target>
+
<target name="ivy-bootstrap" description="Download and install Ivy in the users ant lib dir" depends="ivy-bootstrap1,ivy-bootstrap2,ivy-checksum"/>
<!-- try to download from repo1.maven.org -->
@@ -1084,7 +1092,7 @@ ant test -Dtests.dups=N ...
ant -Dtests.nightly=[false] - nightly test group (@Nightly)
ant -Dtests.weekly=[false] - weekly tests (@Weekly)
ant -Dtests.awaitsfix=[false] - known issue (@AwaitsFix)
-ant -Dtests.slow=[false] - slow tests (@Slow)
+ant -Dtests.slow=[true] - slow tests (@Slow)
#
# Load balancing and caches. --------------------------------------
@@ -1160,7 +1168,7 @@ ${tests-output}/junit4-*.suites - pe
jars so we just order it after compile-test to ensure that -->
<target name="test" depends="clover,compile-test,install-junit4-taskdef,validate" description="Runs unit tests">
<mkdir dir="${junit.output.dir}"/>
- <test-macro threadNum="${tests.jvms}" />
+ <test-macro threadNum="${tests.jvms.override}" />
</target>
<!--
@@ -1276,7 +1284,7 @@ ${tests-output}/junit4-*.suites - pe
<fail message="You must redefine the javadocs task to do something!!!!!"/>
</target>
- <target name="install-maven-tasks" unless="maven-tasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="install-maven-tasks" unless="maven-tasks.uptodate" depends="ivy-availability-check,ivy-configure">
<property name="maven-tasks.uptodate" value="true"/>
<ivy:cachepath organisation="org.apache.maven" module="maven-ant-tasks" revision="2.1.3"
inline="true" conf="master" type="jar" pathid="maven-ant-tasks.classpath"/>
@@ -1534,7 +1542,7 @@ ${tests-output}/junit4-*.suites - pe
</ecj-macro>
</target>
- <target name="-ecj-resolve" unless="ecj.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="-ecj-resolve" unless="ecj.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="org.eclipse.jdt.core.compiler" module="ecj" revision="3.7.2"
inline="true" conf="master" type="jar" pathid="ecj.classpath" />
<componentdef classname="org.eclipse.jdt.core.JDTCompilerAdapter"
@@ -1868,7 +1876,7 @@ ${tests-output}/junit4-*.suites - pe
</macrodef>
<!-- GROOVY scripting engine for ANT tasks -->
- <target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.0.4"
inline="true" conf="default" type="jar" transitive="true" pathid="groovy.classpath"/>
<property name="groovy.loaded" value="true"/>
@@ -1876,8 +1884,8 @@ ${tests-output}/junit4-*.suites - pe
<!-- PEGDOWN macro: Before using depend on the target "resolve-pegdown" -->
- <target name="resolve-pegdown" unless="pegdown.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
- <ivy:cachepath organisation="org.pegdown" module="pegdown" revision="1.1.0"
+ <target name="resolve-pegdown" unless="pegdown.loaded" depends="ivy-availability-check,ivy-configure">
+ <ivy:cachepath organisation="org.pegdown" module="pegdown" revision="1.2.1"
inline="true" conf="default" type="jar" transitive="true" pathid="pegdown.classpath"/>
<property name="pegdown.loaded" value="true"/>
</target>
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java Mon Dec 10 16:36:47 2012
@@ -730,9 +730,6 @@ public class Lucene40TermVectorsReader e
@Override
public Fields get(int docID) throws IOException {
- if (docID < 0 || docID >= numTotalDocs) {
- throw new IllegalArgumentException("doID=" + docID + " is out of bounds [0.." + (numTotalDocs-1) + "]");
- }
if (tvx != null) {
Fields fields = new TVFields(docID);
if (fields.size() == 0) {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java Mon Dec 10 16:36:47 2012
@@ -134,7 +134,6 @@ class VarStraightBytesImpl {
docToAddress[lastDocID] = address + offset;
}
address += numDataBytes; // this is the address after all addr pointers are updated
- iter.close();
} finally {
IOUtils.close(cloneIdx);
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java Mon Dec 10 16:36:47 2012
@@ -33,7 +33,6 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
@@ -52,7 +51,7 @@ import org.apache.lucene.codecs.perfield
// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
// (it writes a minor version, etc).
public class Lucene41Codec extends Codec {
- private final StoredFieldsFormat fieldsFormat = new Lucene40StoredFieldsFormat();
+ private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
private final TermVectorsFormat vectorsFormat = new Lucene40TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat();
private final DocValuesFormat docValuesFormat = new Lucene40DocValuesFormat();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Mon Dec 10 16:36:47 2012
@@ -302,7 +302,7 @@ public class ConcurrentMergeScheduler ex
}
@Override
- public void merge(IndexWriter writer) throws IOException {
+ public synchronized void merge(IndexWriter writer) throws IOException {
assert !Thread.holdsLock(writer);
@@ -328,31 +328,34 @@ public class ConcurrentMergeScheduler ex
// pending merges, until it's empty:
while (true) {
- synchronized(this) {
- long startStallTime = 0;
- while (mergeThreadCount() >= 1+maxMergeCount) {
- startStallTime = System.currentTimeMillis();
- if (verbose()) {
- message(" too many merges; stalling...");
- }
- try {
- wait();
- } catch (InterruptedException ie) {
- throw new ThreadInterruptedException(ie);
- }
- }
-
+ long startStallTime = 0;
+ while (writer.hasPendingMerges() && mergeThreadCount() >= maxMergeCount) {
+ // This means merging has fallen too far behind: we
+ // have already created maxMergeCount threads, and
+ // now there's at least one more merge pending.
+ // Note that only maxThreadCount of
+ // those created merge threads will actually be
+ // running; the rest will be paused (see
+ // updateMergeThreads). We stall this producer
+ // thread to prevent creation of new segments,
+ // until merging has caught up:
+ startStallTime = System.currentTimeMillis();
if (verbose()) {
- if (startStallTime != 0) {
- message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
- }
+ message(" too many merges; stalling...");
+ }
+ try {
+ wait();
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
}
}
+ if (verbose()) {
+ if (startStallTime != 0) {
+ message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
+ }
+ }
- // TODO: we could be careful about which merges to do in
- // the BG (eg maybe the "biggest" ones) vs FG, which
- // merges to do first (the easiest ones?), etc.
MergePolicy.OneMerge merge = writer.getNextMerge();
if (merge == null) {
if (verbose()) {
@@ -361,34 +364,28 @@ public class ConcurrentMergeScheduler ex
return;
}
- // We do this w/ the primary thread to keep
- // deterministic assignment of segment names
- writer.mergeInit(merge);
-
boolean success = false;
try {
- synchronized(this) {
- if (verbose()) {
- message(" consider merge " + writer.segString(merge.segments));
- }
+ if (verbose()) {
+ message(" consider merge " + writer.segString(merge.segments));
+ }
- // OK to spawn a new merge thread to handle this
- // merge:
- final MergeThread merger = getMergeThread(writer, merge);
- mergeThreads.add(merger);
- if (verbose()) {
- message(" launch new thread [" + merger.getName() + "]");
- }
+ // OK to spawn a new merge thread to handle this
+ // merge:
+ final MergeThread merger = getMergeThread(writer, merge);
+ mergeThreads.add(merger);
+ if (verbose()) {
+ message(" launch new thread [" + merger.getName() + "]");
+ }
- merger.start();
+ merger.start();
- // Must call this after starting the thread else
- // the new thread is removed from mergeThreads
- // (since it's not alive yet):
- updateMergeThreads();
+ // Must call this after starting the thread else
+ // the new thread is removed from mergeThreads
+ // (since it's not alive yet):
+ updateMergeThreads();
- success = true;
- }
+ success = true;
} finally {
if (!success) {
writer.mergeFinish(merge);
@@ -482,7 +479,6 @@ public class ConcurrentMergeScheduler ex
// merge that writer says is necessary:
merge = tWriter.getNextMerge();
if (merge != null) {
- tWriter.mergeInit(merge);
updateMergeThreads();
if (verbose()) {
message(" merge thread: do another merge " + tWriter.segString(merge.segments));
@@ -546,4 +542,13 @@ public class ConcurrentMergeScheduler ex
void clearSuppressExceptions() {
suppressExceptions = false;
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(getClass().getSimpleName() + ": ");
+ sb.append("maxThreadCount=").append(maxThreadCount).append(", ");
+ sb.append("maxMergeCount=").append(maxMergeCount).append(", ");
+ sb.append("mergeThreadPriority=").append(mergeThreadPriority);
+ return sb.toString();
+ }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Mon Dec 10 16:36:47 2012
@@ -46,6 +46,7 @@ final class DocumentsWriterFlushControl
private long activeBytes = 0;
private long flushBytes = 0;
private volatile int numPending = 0;
+ private int numDocsSinceStalled = 0; // only with assert
final AtomicBoolean flushDeletes = new AtomicBoolean(false);
private boolean fullFlush = false;
private final Queue<DocumentsWriterPerThread> flushQueue = new LinkedList<DocumentsWriterPerThread>();
@@ -104,8 +105,8 @@ final class DocumentsWriterFlushControl
// 2 * ramBufferBytes -> before we stall we need to cross the 2xRAM Buffer border this is still a valid limit
// (numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) -> those are the total number of DWPT that are not active but not yet fully fluhsed
// all of them could theoretically be taken out of the loop once they crossed the RAM buffer and the last document was the peak delta
- // (perThreadPool.getActiveThreadState() * peakDelta) -> at any given time there could be n threads in flight that crossed the stall control before we reached the limit and each of them could hold a peak document
- final long expected = (2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) + (perThreadPool.getActiveThreadState() * peakDelta);
+ // (numDocsSinceStalled * peakDelta) -> at any given time there could be n threads in flight that crossed the stall control before we reached the limit and each of them could hold a peak document
+ final long expected = (2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) + (numDocsSinceStalled * peakDelta);
// the expected ram consumption is an upper bound at this point and not really the expected consumption
if (peakDelta < (ramBufferBytes >> 1)) {
/*
@@ -183,10 +184,26 @@ final class DocumentsWriterFlushControl
}
return flushingDWPT;
} finally {
- updateStallState();
- assert assertMemory();
+ boolean stalled = updateStallState();
+ assert assertNumDocsSinceStalled(stalled) && assertMemory();
}
}
+
+ private boolean assertNumDocsSinceStalled(boolean stalled) {
+ /*
+ * updates the number of documents "finished" while we are in a stalled state.
+ * this is important for asserting memory upper bounds since it corresponds
+ * to the number of threads that are in-flight and crossed the stall control
+ * check before we actually stalled.
+ * see #assertMemory()
+ */
+ if (stalled) {
+ numDocsSinceStalled++;
+ } else {
+ numDocsSinceStalled = 0;
+ }
+ return true;
+ }
synchronized void doAfterFlush(DocumentsWriterPerThread dwpt) {
assert flushingWriters.containsKey(dwpt);
@@ -204,7 +221,7 @@ final class DocumentsWriterFlushControl
}
}
- private final void updateStallState() {
+ private final boolean updateStallState() {
assert Thread.holdsLock(this);
final long limit = stallLimitBytes();
@@ -219,6 +236,7 @@ final class DocumentsWriterFlushControl
(activeBytes < limit) &&
!closed;
stallControl.updateStalled(stall);
+ return stall;
}
public synchronized void waitForFlush() {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java Mon Dec 10 16:36:47 2012
@@ -107,7 +107,7 @@ public abstract class IndexCommit implem
public abstract long getGeneration();
/** Returns userData, previously passed to {@link
- * IndexWriter#commit(Map)} for this commit. Map is
+ * IndexWriter#setCommitData(Map)} for this commit. Map is
* String -> String. */
public abstract Map<String,String> getUserData() throws IOException;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Mon Dec 10 16:36:47 2012
@@ -540,13 +540,18 @@ public class IndexWriter implements Clos
}
/**
- * Used internally to throw an {@link
- * AlreadyClosedException} if this IndexWriter has been
- * closed.
- * @throws AlreadyClosedException if this IndexWriter is closed
+ * Used internally to throw an {@link AlreadyClosedException} if this
+ * IndexWriter has been closed or is in the process of closing.
+ *
+ * @param failIfClosing
+ * if true, also fail when {@code IndexWriter} is in the process of
+ * closing ({@code closing=true}) but not yet done closing (
+ * {@code closed=false})
+ * @throws AlreadyClosedException
+ * if this IndexWriter is closed or in the process of closing
*/
- protected final void ensureOpen(boolean includePendingClose) throws AlreadyClosedException {
- if (closed || (includePendingClose && closing)) {
+ protected final void ensureOpen(boolean failIfClosing) throws AlreadyClosedException {
+ if (closed || (failIfClosing && closing)) {
throw new AlreadyClosedException("this IndexWriter is closed");
}
}
@@ -554,7 +559,8 @@ public class IndexWriter implements Clos
/**
* Used internally to throw an {@link
* AlreadyClosedException} if this IndexWriter has been
- * closed.
+ * closed ({@code closed=true}) or is in the process of
+ * closing ({@code closing=true}).
* <p>
* Calls {@link #ensureOpen(boolean) ensureOpen(true)}.
* @throws AlreadyClosedException if this IndexWriter is closed
@@ -943,7 +949,7 @@ public class IndexWriter implements Clos
}
if (doFlush) {
- commitInternal(null);
+ commitInternal();
}
if (infoStream.isEnabled("IW")) {
@@ -1407,7 +1413,7 @@ public class IndexWriter implements Clos
*/
public void updateDocument(Term term, IndexDocument doc) throws IOException {
ensureOpen();
- updateDocument(term, doc, getAnalyzer());
+ updateDocument(term, doc, analyzer);
}
/**
@@ -1880,6 +1886,15 @@ public class IndexWriter implements Clos
}
/**
+ * Expert: returns true if there are merges waiting to be scheduled.
+ *
+ * @lucene.experimental
+ */
+ public synchronized boolean hasPendingMerges() {
+ return pendingMerges.size() != 0;
+ }
+
+ /**
* Close the <code>IndexWriter</code> without committing
* any changes that have occurred since the last commit
* (or since it was opened, if commit hasn't been called).
@@ -2063,7 +2078,7 @@ public class IndexWriter implements Clos
// they are aborted.
while(runningMerges.size() > 0) {
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge to abort");
+ infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge/s to abort");
}
doWait();
}
@@ -2537,20 +2552,7 @@ public class IndexWriter implements Clos
*/
protected void doBeforeFlush() throws IOException {}
- /** Expert: prepare for commit.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
- *
- * @see #prepareCommit(Map) */
- public final void prepareCommit() throws IOException {
- ensureOpen();
- prepareCommit(null);
- }
-
- /** <p>Expert: prepare for commit, specifying
- * commitUserData Map (String -> String). This does the
+ /** <p>Expert: prepare for commit. This does the
* first phase of 2-phase commit. This method does all
* steps necessary to commit changes since this writer
* was opened: flushes pending added and deleted docs,
@@ -2560,29 +2562,22 @@ public class IndexWriter implements Clos
* #rollback()} to revert the commit and undo all changes
* done since the writer was opened.</p>
*
- * <p>You can also just call {@link #commit(Map)} directly
+ * <p>You can also just call {@link #commit()} directly
* without prepareCommit first in which case that method
* will internally call prepareCommit.
*
* <p><b>NOTE</b>: if this method hits an OutOfMemoryError
* you should immediately close the writer. See <a
* href="#OOME">above</a> for details.</p>
- *
- * @param commitUserData Opaque Map (String->String)
- * that's recorded into the segments file in the index,
- * and retrievable by {@link
- * IndexCommit#getUserData}. Note that when
- * IndexWriter commits itself during {@link #close}, the
- * commitUserData is unchanged (just carried over from
- * the prior commit). If this is null then the previous
- * commitUserData is kept. Also, the commitUserData will
- * only "stick" if there are actually changes in the
- * index to commit.
*/
- public final void prepareCommit(Map<String,String> commitUserData) throws IOException {
- ensureOpen(false);
+ public final void prepareCommit() throws IOException {
+ ensureOpen();
+ prepareCommitInternal();
+ }
+ private void prepareCommitInternal() throws IOException {
synchronized(commitLock) {
+ ensureOpen(false);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "prepareCommit: flush");
infoStream.message("IW", " index before flush " + segString());
@@ -2672,10 +2667,33 @@ public class IndexWriter implements Clos
}
}
- startCommit(toCommit, commitUserData);
+ startCommit(toCommit);
}
}
-
+
+ /**
+ * Sets the commit user data map. That method is considered a transaction by
+ * {@link IndexWriter} and will be {@link #commit() committed} even if no other
+ * changes were made to the writer instance. Note that you must call this method
+ * before {@link #prepareCommit()}, or otherwise it won't be included in the
+ * follow-on {@link #commit()}.
+ * <p>
+ * <b>NOTE:</b> the map is cloned internally, therefore altering the map's
+ * contents after calling this method has no effect.
+ */
+ public final synchronized void setCommitData(Map<String,String> commitUserData) {
+ segmentInfos.setUserData(new HashMap<String,String>(commitUserData));
+ ++changeCount;
+ }
+
+ /**
+ * Returns the commit user data map that was last committed, or the one that
+ * was set on {@link #setCommitData(Map)}.
+ */
+ public final synchronized Map<String,String> getCommitData() {
+ return segmentInfos.getUserData();
+ }
+
// Used only by commit and prepareCommit, below; lock
// order is commitLock -> IW
private final Object commitLock = new Object();
@@ -2708,29 +2726,13 @@ public class IndexWriter implements Clos
* href="#OOME">above</a> for details.</p>
*
* @see #prepareCommit
- * @see #commit(Map)
*/
public final void commit() throws IOException {
- commit(null);
- }
-
- /** Commits all changes to the index, specifying a
- * commitUserData Map (String -> String). This just
- * calls {@link #prepareCommit(Map)} (if you didn't
- * already call it) and then {@link #commit}.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
- */
- public final void commit(Map<String,String> commitUserData) throws IOException {
-
ensureOpen();
-
- commitInternal(commitUserData);
+ commitInternal();
}
- private final void commitInternal(Map<String,String> commitUserData) throws IOException {
+ private final void commitInternal() throws IOException {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commit: start");
@@ -2747,7 +2749,7 @@ public class IndexWriter implements Clos
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commit: now prepare");
}
- prepareCommit(commitUserData);
+ prepareCommitInternal();
} else {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commit: already prepared");
@@ -2771,7 +2773,6 @@ public class IndexWriter implements Clos
}
lastCommitChangeCount = pendingCommitChangeCount;
segmentInfos.updateGeneration(pendingCommit);
- segmentInfos.setUserData(pendingCommit.getUserData());
rollbackSegments = pendingCommit.createBackupSegmentInfos();
deleter.checkpoint(pendingCommit, true);
} finally {
@@ -3890,7 +3891,7 @@ public class IndexWriter implements Clos
* if it wasn't already. If that succeeds, then we
* prepare a new segments_N file but do not fully commit
* it. */
- private void startCommit(final SegmentInfos toSync, final Map<String,String> commitUserData) throws IOException {
+ private void startCommit(final SegmentInfos toSync) throws IOException {
assert testPoint("startStartCommit");
assert pendingCommit == null;
@@ -3923,10 +3924,6 @@ public class IndexWriter implements Clos
}
assert filesExist(toSync);
-
- if (commitUserData != null) {
- toSync.setUserData(commitUserData);
- }
}
assert testPoint("midStartCommit");