You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by ry...@apache.org on 2007/05/11 00:10:06 UTC
svn commit: r537014 - in /lucene/solr/trunk: CHANGES.txt
lib/commons-codec-1.3.jar
src/java/org/apache/solr/analysis/PhoneticFilter.java
src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
src/test/org/apache/solr/analysis/TestPhoneticFilter.java
Author: ryan
Date: Thu May 10 15:10:05 2007
New Revision: 537014
URL: http://svn.apache.org/viewvc?view=rev&rev=537014
Log:
SOLR-224: Adding a PhoneticFilterFactory that uses commons codec language encoders to build phonetically similar tokens.
Added:
lucene/solr/trunk/lib/commons-codec-1.3.jar (with props)
lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java (with props)
lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java (with props)
lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?view=diff&rev=537014&r1=537013&r2=537014
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Thu May 10 15:10:05 2007
@@ -184,6 +184,10 @@
30. SOLR-226: Added support for dynamic field as the destination of a
copyField using glob (*) replacement. (ryan)
+31. SOLR-224: Adding a PhoneticFilterFactory that uses apache commons codec
+ language encoders to build phonetically similar tokens. This currently
+ supports: DoubleMetaphone, Metaphone, Soundex, and RefinedSoundex (ryan)
+
Changes in runtime behavior
1. Highlighting using DisMax will only pick up terms from the main
user query, not boost or filter queries (klaas).
Added: lucene/solr/trunk/lib/commons-codec-1.3.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/commons-codec-1.3.jar?view=auto&rev=537014
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/lib/commons-codec-1.3.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java?view=auto&rev=537014
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java (added)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java Thu May 10 15:10:05 2007
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.analysis;
+
+import org.apache.commons.codec.Encoder;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+
+import java.io.IOException;
+
+/**
+ * Create tokens for phonetic matches. See:
+ * http://jakarta.apache.org/commons/codec/api-release/org/apache/commons/codec/language/package-summary.html
+ *
+ * @version $Id:$
+ */
+public class PhoneticFilter extends TokenFilter
+{
+ protected boolean inject = true;
+ protected Encoder encoder = null;
+ protected String name = null;
+
+ protected Token save = null;
+
+ public PhoneticFilter(TokenStream in, Encoder encoder, String name, boolean inject) {
+ super(in);
+ this.encoder = encoder;
+ this.name = name;
+ this.inject = inject;
+ }
+
+ @Override
+ public final Token next() throws IOException {
+ if( save != null ) {
+ Token temp = save;
+ save = null;
+ return temp;
+ }
+
+ Token t = input.next();
+ if( t != null ) {
+ String value = t.termText();
+ try {
+ value = encoder.encode(t.termText()).toString();
+ }
+ catch (Exception ignored) {} // just use the direct text
+
+ Token m = new Token(value, t.startOffset(), t.endOffset(), name );
+ if( inject ) {
+ m.setPositionIncrement(0);
+ save = m;
+ }
+ else {
+ // replace the token rather then add it too the stream
+ return m;
+ }
+ }
+ return t;
+ }
+}
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilter.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java?view=auto&rev=537014
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java (added)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java Thu May 10 15:10:05 2007
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.analysis;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.codec.Encoder;
+import org.apache.commons.codec.language.DoubleMetaphone;
+import org.apache.commons.codec.language.Metaphone;
+import org.apache.commons.codec.language.RefinedSoundex;
+import org.apache.commons.codec.language.Soundex;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.solr.core.SolrException;
+
+/**
+ * Create tokens based on phonetic encoders
+ *
+ * http://jakarta.apache.org/commons/codec/api-release/org/apache/commons/codec/language/package-summary.html
+ *
+ * This takes two arguments:
+ * "encoder" required, one of "DoubleMetaphone", "Metaphone", "Soundex", "RefinedSoundex"
+ *
+ * "inject" (default=true) add tokens to the stream with the offset=0
+ *
+ * @version $Id:$
+ * @see PhoneticFilter
+ */
+public class PhoneticFilterFactory extends BaseTokenFilterFactory
+{
+ public static final String ENCODER = "encoder";
+ public static final String INJECT = "inject"; // boolean
+
+ private static final Map<String, Class<? extends Encoder>> registry;
+ static {
+ registry = new HashMap<String, Class<? extends Encoder>>();
+ registry.put( "DoubleMetaphone".toUpperCase(), DoubleMetaphone.class );
+ registry.put( "Metaphone".toUpperCase(), Metaphone.class );
+ registry.put( "Soundex".toUpperCase(), Soundex.class );
+ registry.put( "RefinedSoundex".toUpperCase(), RefinedSoundex.class );
+ }
+
+ protected boolean inject = true;
+ protected String name = null;
+ protected Encoder encoder = null;
+
+ @Override
+ public void init(Map<String,String> args) {
+ super.init( args );
+
+ if( args.get( "inject" ) != null ) {
+ inject = Boolean.getBoolean( args.get( INJECT ) );
+ }
+
+ String name = args.get( ENCODER );
+ if( name == null ) {
+ throw new SolrException( 500, "Missing required parameter: "+ENCODER
+ +" ["+registry.keySet()+"]" );
+ }
+ Class<? extends Encoder> clazz = registry.get(name.toUpperCase());
+ if( clazz == null ) {
+ throw new SolrException( 500, "Unknown encoder: "+name +" ["+registry.keySet()+"]" );
+ }
+
+ try {
+ encoder = clazz.newInstance();
+ }
+ catch (Exception e) {
+ throw new SolrException( 500, "Error initializing: "+name + "/"+clazz, e );
+ }
+ }
+
+ public TokenStream create(TokenStream input) {
+ return new PhoneticFilter(input,encoder,name,inject);
+ }
+}
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java?view=auto&rev=537014
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java (added)
+++ lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java Thu May 10 15:10:05 2007
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.analysis;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.codec.Encoder;
+import org.apache.commons.codec.language.DoubleMetaphone;
+import org.apache.commons.codec.language.Metaphone;
+import org.apache.commons.codec.language.RefinedSoundex;
+import org.apache.commons.codec.language.Soundex;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+
+
+/**
+ * @version $Id:$
+ */
+public class TestPhoneticFilter extends TestCase {
+
+ public void testFactory()
+ {
+ Map<String,String> args = new HashMap<String, String>();
+
+ PhoneticFilterFactory ff = new PhoneticFilterFactory();
+ try {
+ ff.init( args );
+ fail( "missing encoder parameter" );
+ }
+ catch( Exception ex ) {}
+ args.put( PhoneticFilterFactory.ENCODER, "XXX" );
+ try {
+ ff.init( args );
+ fail( "unknown encoder parameter" );
+ }
+ catch( Exception ex ) {}
+
+ args.put( PhoneticFilterFactory.ENCODER, "Metaphone" );
+ ff.init( args );
+ assertTrue( ff.encoder instanceof Metaphone );
+ assertTrue( ff.inject ); // default
+
+ args.put( PhoneticFilterFactory.INJECT, "false" );
+ ff.init( args );
+ assertFalse( ff.inject );
+ }
+
+ public void runner( Encoder enc, boolean inject ) throws Exception
+ {
+ String[] input = new String[] {
+ "aaa", "bbb", "ccc", "easgasg"
+ };
+
+ ArrayList<Token> stream = new ArrayList<Token>();
+ ArrayList<Token> output = new ArrayList<Token>();
+ for( String s : input ) {
+ stream.add( new Token( s, 0, s.length() ) );
+ if( inject ) {
+ output.add( new Token( s, 0, s.length() ) );
+ }
+ output.add( new Token( enc.encode(s).toString(), 0, s.length() ) );
+ }
+
+ PhoneticFilter filter = new PhoneticFilter(
+ new IterTokenStream(stream.iterator()), enc, "text", inject );
+
+ for( Token t : output ) {
+ Token got = filter.next();
+ assertEquals( t.termText(), got.termText());
+ }
+ assertNull( filter.next() ); // no more tokens
+ }
+
+ public void testEncodes() throws Exception {
+ runner( new DoubleMetaphone(), true );
+ runner( new Metaphone(), true );
+ runner( new Soundex(), true );
+ runner( new RefinedSoundex(), true );
+
+ runner( new DoubleMetaphone(), false );
+ runner( new Metaphone(), false );
+ runner( new Soundex(), false );
+ runner( new RefinedSoundex(), false );
+ }
+
+ public static class IterTokenStream extends TokenStream {
+ Iterator<Token> toks;
+ public IterTokenStream(Iterator<Token> toks) {
+ this.toks = toks;
+ }
+ public Token next() {
+ if (toks.hasNext()) {
+ return toks.next();
+ }
+ return null;
+ }
+ }
+}
Propchange: lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/test/org/apache/solr/analysis/TestPhoneticFilter.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL