You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by gs...@apache.org on 2008/02/24 03:34:16 UTC
svn commit: r630572 - in /lucene/solr/trunk: CHANGES.txt
src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
src/test/org/apache/solr/analysis/TestCapitalizationFilter.java
Author: gsingers
Date: Sat Feb 23 18:34:15 2008
New Revision: 630572
URL: http://svn.apache.org/viewvc?rev=630572&view=rev
Log:
SOLR-468: Changed semantics of CapitalizationFilterFactory to keep the original token, not the value in the map
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
lucene/solr/trunk/src/test/org/apache/solr/analysis/TestCapitalizationFilter.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=630572&r1=630571&r2=630572&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Sat Feb 23 18:34:15 2008
@@ -106,6 +106,8 @@
17. SOLR-248: Added CapitalizationFilterFactory that creates tokens with
normalized capitalization. This filter is useful for facet display,
but will not work with a prefix query. (ryan)
+ SOLR-468: Change to the semantics to keep the original token, not the token in the Map. Also switched to
+ use Lucene's new reusable token capabilities. (gsingers)
18. SOLR-307: Added NGramFilterFactory and EdgeNGramFilterFactory.
(Thomas Peuss via Otis Gospodnetic)
Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java?rev=630572&r1=630571&r2=630572&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java Sat Feb 23 18:34:15 2008
@@ -17,199 +17,226 @@
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
+import java.util.Collections;
import java.util.Map;
import java.util.StringTokenizer;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.solr.core.SolrConfig;
-import org.apache.solr.analysis.BaseTokenFilterFactory;
-
/**
* A filter to apply normal capitalization rules to Tokens. It will make the first letter
- * capital and the rest lower case.
- *
+ * capital and the rest lower case.
+ * <p/>
* This filter is particularly useful to build nice looking facet parameters. This filter
* is not appropriate if you intend to use a prefix query.
- *
- * The factory takes parameters:
- * "onlyFirstWord" - should each word be capitalized or all of the words?
- * "keep" - a keep word list. Each word that should be kept separated by whitespace.
+ * <p/>
+ * The factory takes parameters:<br/>
+ * "onlyFirstWord" - should each word be capitalized or all of the words?<br/>
+ * "keep" - a keep word list. Each word that should be kept separated by whitespace.<br/>
+ * "keepIgnoreCase - true or false. If true, the keep list will be considered case-insensitive.
+ * "forceFirstLetter" - Force the first letter to be capitalized even if it is in the keep list<br/>
* "okPrefix" - do not change word capitalization if a word begins with something in this list.
- * for example if "McK" is on the okPrefix list, the word "McKinley" should not be changed to
- * "Mckinley"
- * "minWordLength" - how long the word needs to be to get capitalization applied. If the
- * minWordLength is 3, "and" > "And" but "or" stays "or"
+ * for example if "McK" is on the okPrefix list, the word "McKinley" should not be changed to
+ * "Mckinley"<br/>
+ * "minWordLength" - how long the word needs to be to get capitalization applied. If the
+ * minWordLength is 3, "and" > "And" but "or" stays "or"<br/>
* "maxWordCount" - if the token contains more then maxWordCount words, the capitalization is
- * assumed to be correct.
- *
- * @since solr 1.3
+ * assumed to be correct.<br/>
+ *
* @version $Id$
+ * @since solr 1.3
*/
-public class CapitalizationFilterFactory extends BaseTokenFilterFactory
-{
+public class CapitalizationFilterFactory extends BaseTokenFilterFactory {
+ public static final int DEFAULT_MAX_WORD_COUNT = Integer.MAX_VALUE;
public static final String KEEP = "keep";
+ public static final String KEEP_IGNORE_CASE = "keepIgnoreCase";
public static final String OK_PREFIX = "okPrefix";
public static final String MIN_WORD_LENGTH = "minWordLength";
public static final String MAX_WORD_COUNT = "maxWordCount";
public static final String MAX_TOKEN_LENGTH = "maxTokenLength";
public static final String ONLY_FIRST_WORD = "onlyFirstWord";
public static final String FORCE_FIRST_LETTER = "forceFirstLetter";
-
- Map<String,String> keep = new HashMap<String, String>(); // not synchronized because it is only initialized once
-
- Collection<String> okPrefix = new ArrayList<String>(); // for Example: McK
-
+
+ //Map<String,String> keep = new HashMap<String, String>(); // not synchronized because it is only initialized once
+ CharArraySet keep;
+
+ Collection<char[]> okPrefix = Collections.emptyList(); // for Example: McK
+
int minWordLength = 0; // don't modify capitalization for words shorter then this
- int maxWordCount = Integer.MAX_VALUE;
- int maxTokenLength = Integer.MAX_VALUE;
+ int maxWordCount = DEFAULT_MAX_WORD_COUNT;
+ int maxTokenLength = DEFAULT_MAX_WORD_COUNT;
boolean onlyFirstWord = true;
boolean forceFirstLetter = true; // make sure the first letter is capitol even if it is in the keep list
-
+
@Override
- public void init(Map<String,String> args) {
- super.init( args );
-
- String k = args.get( KEEP );
- if( k != null ) {
- StringTokenizer st = new StringTokenizer( k );
- while( st.hasMoreTokens() ) {
+ public void init(Map<String, String> args) {
+ super.init(args);
+
+ String k = args.get(KEEP);
+ if (k != null) {
+ StringTokenizer st = new StringTokenizer(k);
+ boolean ignoreCase = false;
+ String ignoreStr = args.get(KEEP_IGNORE_CASE);
+ if ("true".equalsIgnoreCase(ignoreStr)) {
+ ignoreCase = true;
+ }
+ keep = new CharArraySet(10, ignoreCase);
+ while (st.hasMoreTokens()) {
k = st.nextToken().trim();
- keep.put( k.toUpperCase(), k );
+ keep.add(k.toCharArray());
}
}
-
- k = args.get( OK_PREFIX );
- if( k != null ) {
- StringTokenizer st = new StringTokenizer( k );
- while( st.hasMoreTokens() ) {
- okPrefix.add( st.nextToken().trim() );
+
+ k = args.get(OK_PREFIX);
+ if (k != null) {
+ okPrefix = new ArrayList<char[]>();
+ StringTokenizer st = new StringTokenizer(k);
+ while (st.hasMoreTokens()) {
+ okPrefix.add(st.nextToken().trim().toCharArray());
}
}
-
- k = args.get( MIN_WORD_LENGTH );
- if( k != null ) {
- minWordLength = Integer.valueOf( k );
+
+ k = args.get(MIN_WORD_LENGTH);
+ if (k != null) {
+ minWordLength = Integer.valueOf(k);
}
- k = args.get( MAX_WORD_COUNT );
- if( k != null ) {
- maxWordCount = Integer.valueOf( k );
+ k = args.get(MAX_WORD_COUNT);
+ if (k != null) {
+ maxWordCount = Integer.valueOf(k);
}
- k = args.get( MAX_TOKEN_LENGTH );
- if( k != null ) {
- maxTokenLength = Integer.valueOf( k );
+ k = args.get(MAX_TOKEN_LENGTH);
+ if (k != null) {
+ maxTokenLength = Integer.valueOf(k);
}
- k = args.get( ONLY_FIRST_WORD );
- if( k != null ) {
- onlyFirstWord = Boolean.valueOf( k );
+ k = args.get(ONLY_FIRST_WORD);
+ if (k != null) {
+ onlyFirstWord = Boolean.valueOf(k);
}
- k = args.get( FORCE_FIRST_LETTER );
- if( k != null ) {
- forceFirstLetter = Boolean.valueOf( k );
+ k = args.get(FORCE_FIRST_LETTER);
+ if (k != null) {
+ forceFirstLetter = Boolean.valueOf(k);
}
}
-
- public String processWord( String w, int wordCount )
- {
- if( w.length() < 1 ) {
- return w;
- }
- if( onlyFirstWord && wordCount > 0 ) {
- return w.toLowerCase();
+
+
+ public void processWord(char[] buffer, int offset, int length, int wordCount) {
+ if (length < 1) {
+ return;
+ }
+ if (onlyFirstWord && wordCount > 0) {
+ for (int i = 0; i < length; i++) {
+ buffer[offset + i] = Character.toLowerCase(buffer[offset + i]);
+
+ }
+ return;
}
-
- String k = keep.get( w.toUpperCase() );
- if( k != null ) {
- if( wordCount == 0 && forceFirstLetter && Character.isLowerCase( k.charAt(0) ) ) {
- return Character.toUpperCase( k.charAt(0) ) + k.substring( 1 );
+
+ if (keep.contains(buffer, offset, length)) {
+ if (wordCount == 0 && forceFirstLetter) {
+ buffer[offset] = Character.toUpperCase(buffer[offset]);
}
- return k;
+ return;
}
- if( w.length() < minWordLength ) {
- return w;
+ if (length < minWordLength) {
+ return;
}
- for( String prefix : okPrefix ) {
- if( w.startsWith( prefix ) ) {
- return w;
+ for (char[] prefix : okPrefix) {
+ if (length >= prefix.length) { //don't bother checking if the buffer length is less than the prefix
+ boolean match = true;
+ for (int i = 0; i < prefix.length; i++) {
+ if (prefix[i] != buffer[offset + i]) {
+ match = false;
+ break;
+ }
+ }
+ if (match == true) {
+ return;
+ }
}
}
-
+
// We know it has at least one character
- char[] chars = w.toCharArray();
+ /*char[] chars = w.toCharArray();
StringBuilder word = new StringBuilder( w.length() );
- word.append( Character.toUpperCase( chars[0] ) );
- for( int i=1; i<chars.length; i++ ) {
- word.append( Character.toLowerCase( chars[i] ) );
+ word.append( Character.toUpperCase( chars[0] ) );*/
+ buffer[offset] = Character.toUpperCase(buffer[offset]);
+
+ for (int i = 1; i < length; i++) {
+ buffer[offset + i] = Character.toLowerCase(buffer[offset + i]);
}
- return word.toString();
+ //return word.toString();
}
-
+
public CapitalizationFilter create(TokenStream input) {
- return new CapitalizationFilter(input,this);
+ return new CapitalizationFilter(input, this);
}
}
-
/**
* This relies on the Factory so that the difficult stuff does not need to be
* re-initialized each time the filter runs.
- *
+ * <p/>
* This is package protected since it is not useful without the Factory
*/
-class CapitalizationFilter extends TokenFilter
-{
+class CapitalizationFilter extends TokenFilter {
protected final CapitalizationFilterFactory factory;
-
- public CapitalizationFilter(TokenStream in, final CapitalizationFilterFactory factory ) {
+
+ public CapitalizationFilter(TokenStream in, final CapitalizationFilterFactory factory) {
super(in);
this.factory = factory;
}
-
+
@Override
- public final Token next() throws IOException {
-
- Token t = input.next();
- if( t != null ) {
- String s = t.termText();
- if( s.length() < factory.maxTokenLength ) {
+ public Token next(Token token) throws IOException {
+ Token t = input.next(token);
+ if (t != null) {
+
+ char[] termBuffer = t.termBuffer();
+ int termBufferLength = t.termLength();
+ char[] backup = null;
+ if (factory.maxWordCount < CapitalizationFilterFactory.DEFAULT_MAX_WORD_COUNT) {
+ //make a backup in case we exceed the word count
+ System.arraycopy(termBuffer, 0, backup, 0, termBufferLength);
+ }
+ if (termBuffer.length < factory.maxTokenLength) {
int wordCount = 0;
- StringBuilder word = new StringBuilder( s.length() );
- StringBuilder text = new StringBuilder( s.length() );
- for( char c : s.toCharArray() ) {
- if( c <= ' ' || c == '.' ) {
- if( word.length() > 0 ) {
- text.append( factory.processWord( word.toString(), wordCount++ ) );
- word.setLength( 0 );
+ int lastWordStart = 0;
+ for (int i = 0; i < termBufferLength; i++) {
+ char c = termBuffer[i];
+ if (c <= ' ' || c == '.') {
+ int len = i - lastWordStart;
+ if (len > 0) {
+ factory.processWord(termBuffer, lastWordStart, len, wordCount++);
+ lastWordStart = i + 1;
+ i++;
}
- text.append( c );
- }
- else {
- word.append( c );
}
}
-
- // Add the last word
- if( word.length() > 0 ) {
- text.append( factory.processWord( word.toString(), wordCount++ ) );
+
+ // process the last word
+ if (lastWordStart < termBuffer.length) {
+ factory.processWord(termBuffer, lastWordStart, termBuffer.length - lastWordStart, wordCount++);
}
-
- if( wordCount <= factory.maxWordCount ) {
- t.setTermText( text.toString() );
+
+ if (wordCount > factory.maxWordCount) {
+ t.setTermBuffer(backup, 0, termBufferLength);
}
}
}
return t;
}
+
}
Modified: lucene/solr/trunk/src/test/org/apache/solr/analysis/TestCapitalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/analysis/TestCapitalizationFilter.java?rev=630572&r1=630571&r2=630572&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/analysis/TestCapitalizationFilter.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/analysis/TestCapitalizationFilter.java Sat Feb 23 18:34:15 2008
@@ -34,14 +34,33 @@
CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
factory.init( args );
+ char[] termBuffer;
+ termBuffer = "kiTTEN".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "Kitten", new String(termBuffer, 0, termBuffer.length));
- assertEquals( "Kitten", factory.processWord( "kiTTEN", 0 ) );
factory.forceFirstLetter = true;
- assertEquals( "And", factory.processWord( "AnD", 0 ) ); // first is forced
+
+ termBuffer = "and".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "And", new String(termBuffer, 0, termBuffer.length));//first is forced
+
+ termBuffer = "AnD".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "And", new String(termBuffer, 0, termBuffer.length));//first is forced, but it's not a keep word, either
+
factory.forceFirstLetter = false;
- assertEquals( "and", factory.processWord( "AnD", 0 ) ); // first is forced
+ termBuffer = "AnD".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "And", new String(termBuffer, 0, termBuffer.length)); //first is not forced, but it's not a keep word, either
+
factory.forceFirstLetter = true;
- assertEquals( "BIG", factory.processWord( "big", 0 ) );
+ termBuffer = "big".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "Big", new String(termBuffer, 0, termBuffer.length));
+ termBuffer = "BIG".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "BIG", new String(termBuffer, 0, termBuffer.length));
String out = tsToString( factory.create( new IterTokenStream( "Hello thEre my Name is Ryan" ) ) );
assertEquals( "Hello there my name is ryan", out );
@@ -74,7 +93,27 @@
assertEquals( "1st 2nd Third", out );
factory.forceFirstLetter = true;
- out = tsToString( factory.create( new IterTokenStream( "the The" ) ) );
- assertEquals( "The the", out );
+ out = tsToString( factory.create( new IterTokenStream( "the The the" ) ) );
+ assertEquals( "The The the", out );
+ }
+
+ public void testKeepIgnoreCase() throws Exception {
+ Map<String,String> args = new HashMap<String, String>();
+ args.put( CapitalizationFilterFactory.KEEP, "kitten" );
+ args.put( CapitalizationFilterFactory.KEEP_IGNORE_CASE, "true" );
+ args.put( CapitalizationFilterFactory.ONLY_FIRST_WORD, "true" );
+
+ CapitalizationFilterFactory factory = new CapitalizationFilterFactory();
+ factory.init( args );
+ char[] termBuffer;
+ termBuffer = "kiTTEN".toCharArray();
+ factory.forceFirstLetter = true;
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "KiTTEN", new String(termBuffer, 0, termBuffer.length));
+
+ factory.forceFirstLetter = false;
+ termBuffer = "kiTTEN".toCharArray();
+ factory.processWord(termBuffer, 0, termBuffer.length, 0 );
+ assertEquals( "kiTTEN", new String(termBuffer, 0, termBuffer.length));
}
}