You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by gs...@apache.org on 2008/01/24 16:42:53 UTC
svn commit: r614905 -
/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
Author: gsingers
Date: Thu Jan 24 07:42:50 2008
New Revision: 614905
URL: http://svn.apache.org/viewvc?rev=614905&view=rev
Log:
Added javadocs about UNTOKENIZED_TOKEN_FLAG conundrum
Modified:
lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=614905&r1=614904&r2=614905&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (original)
+++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java Thu Jan 24 07:42:50 2008
@@ -49,7 +49,9 @@
public static final int TOKENS_ONLY = 0;
public static final int UNTOKENIZED_ONLY = 1;
public static final int BOTH = 2;
-
+ /**
+ * This flag is used to indicate that the produced "Token" would, if {@link #TOKENS_ONLY} was used, produce multiple tokens.
+ */
public static final int UNTOKENIZED_TOKEN_FLAG = 1;
/**
* A private instance of the JFlex-constructed scanner