You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/10/30 02:00:07 UTC
svn commit: r1195082 - in /lucene/dev/trunk: dev-tools/eclipse/
dev-tools/maven/ modules/analysis/phonetic/lib/ solr/
solr/core/src/java/org/apache/solr/analysis/
solr/core/src/test/org/apache/solr/analysis/ solr/lib/
Author: rmuir
Date: Sun Oct 30 01:00:06 2011
New Revision: 1195082
URL: http://svn.apache.org/viewvc?rev=1195082&view=rev
Log:
SOLR-2276: Support for cologne phonetic
Added:
lucene/dev/trunk/modules/analysis/phonetic/lib/commons-codec-1.5.jar (with props)
lucene/dev/trunk/solr/lib/commons-codec-1.5.jar (with props)
Removed:
lucene/dev/trunk/modules/analysis/phonetic/lib/commons-codec-1.4.jar
lucene/dev/trunk/solr/lib/commons-codec-1.4.jar
Modified:
lucene/dev/trunk/dev-tools/eclipse/dot.classpath
lucene/dev/trunk/dev-tools/maven/pom.xml.template
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java
Modified: lucene/dev/trunk/dev-tools/eclipse/dot.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-tools/eclipse/dot.classpath?rev=1195082&r1=1195081&r2=1195082&view=diff
==============================================================================
--- lucene/dev/trunk/dev-tools/eclipse/dot.classpath (original)
+++ lucene/dev/trunk/dev-tools/eclipse/dot.classpath Sun Oct 30 01:00:06 2011
@@ -87,7 +87,7 @@
<classpathentry kind="lib" path="lucene/lib/junit-4.7.jar"/>
<classpathentry kind="lib" path="lucene/contrib/sandbox/lib/jakarta-regexp-1.4.jar"/>
<classpathentry kind="lib" path="modules/analysis/icu/lib/icu4j-4_8_1_1.jar"/>
- <classpathentry kind="lib" path="modules/analysis/phonetic/lib/commons-codec-1.4.jar"/>
+ <classpathentry kind="lib" path="modules/analysis/phonetic/lib/commons-codec-1.5.jar"/>
<classpathentry kind="lib" path="modules/analysis/morfologik/lib/morfologik-fsa-1.5.2.jar"/>
<classpathentry kind="lib" path="modules/analysis/morfologik/lib/morfologik-polish-1.5.2.jar"/>
<classpathentry kind="lib" path="modules/analysis/morfologik/lib/morfologik-stemming-1.5.2.jar"/>
@@ -98,7 +98,7 @@
<classpathentry kind="lib" path="modules/benchmark/lib/commons-logging-1.0.4.jar"/>
<classpathentry kind="lib" path="modules/benchmark/lib/xercesImpl-2.9.1-patched-XERCESJ-1257.jar"/>
<classpathentry kind="lib" path="solr/lib/apache-solr-noggit-r1099557.jar"/>
- <classpathentry kind="lib" path="solr/lib/commons-codec-1.4.jar"/>
+ <classpathentry kind="lib" path="solr/lib/commons-codec-1.5.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-fileupload-1.2.1.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-httpclient-3.1.jar"/>
Modified: lucene/dev/trunk/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev-tools/maven/pom.xml.template?rev=1195082&r1=1195081&r2=1195082&view=diff
==============================================================================
--- lucene/dev/trunk/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/trunk/dev-tools/maven/pom.xml.template Sun Oct 30 01:00:06 2011
@@ -138,7 +138,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.4</version>
+ <version>1.5</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
Added: lucene/dev/trunk/modules/analysis/phonetic/lib/commons-codec-1.5.jar
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/phonetic/lib/commons-codec-1.5.jar?rev=1195082&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1195082&r1=1195081&r2=1195082&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sun Oct 30 01:00:06 2011
@@ -380,6 +380,9 @@ New Features
* SOLR-2818: Added before/after count response parsing support for range facets in
SolrJ. (Bernhard Frauendienst via Martijn van Groningen)
+
+* SOLR-2276: Add support for cologne phonetic to PhoneticFilterFactory.
+ (Marc Pompl via rmuir)
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java?rev=1195082&r1=1195081&r2=1195082&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java Sun Oct 30 01:00:06 2011
@@ -21,9 +21,12 @@ import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.codec.Encoder;
import org.apache.commons.codec.language.Caverphone;
+import org.apache.commons.codec.language.ColognePhonetic;
import org.apache.commons.codec.language.DoubleMetaphone;
import org.apache.commons.codec.language.Metaphone;
import org.apache.commons.codec.language.RefinedSoundex;
@@ -59,16 +62,18 @@ public class PhoneticFilterFactory exten
{
public static final String ENCODER = "encoder";
public static final String INJECT = "inject"; // boolean
+ private static final String PACKAGE_CONTAINIG_ENCODERS = "org.apache.commons.codec.language.";
- private static final Map<String, Class<? extends Encoder>> registry;
- static {
- registry = new HashMap<String, Class<? extends Encoder>>();
- registry.put( "DoubleMetaphone".toUpperCase(Locale.ENGLISH), DoubleMetaphone.class );
- registry.put( "Metaphone".toUpperCase(Locale.ENGLISH), Metaphone.class );
- registry.put( "Soundex".toUpperCase(Locale.ENGLISH), Soundex.class );
- registry.put( "RefinedSoundex".toUpperCase(Locale.ENGLISH), RefinedSoundex.class );
- registry.put( "Caverphone".toUpperCase(Locale.ENGLISH), Caverphone.class );
- }
+ private static final Map<String, Class<? extends Encoder>> registry = new HashMap<String, Class<? extends Encoder>>()
+ {{
+ put( "DoubleMetaphone".toUpperCase(Locale.ENGLISH), DoubleMetaphone.class );
+ put( "Metaphone".toUpperCase(Locale.ENGLISH), Metaphone.class );
+ put( "Soundex".toUpperCase(Locale.ENGLISH), Soundex.class );
+ put( "RefinedSoundex".toUpperCase(Locale.ENGLISH), RefinedSoundex.class );
+ put( "Caverphone".toUpperCase(Locale.ENGLISH), Caverphone.class );
+ put( "ColognePhonetic".toUpperCase(Locale.ENGLISH), ColognePhonetic.class );
+ }};
+ private static final Lock lock = new ReentrantLock();
protected boolean inject = true;
protected String name = null;
@@ -87,7 +92,12 @@ public class PhoneticFilterFactory exten
}
Class<? extends Encoder> clazz = registry.get(name.toUpperCase(Locale.ENGLISH));
if( clazz == null ) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown encoder: "+name +" ["+registry.keySet()+"]" );
+ lock.lock();
+ try {
+ clazz = resolveEncoder(name);
+ } finally {
+ lock.unlock();
+ }
}
try {
@@ -105,6 +115,30 @@ public class PhoneticFilterFactory exten
}
}
+ private Class<? extends Encoder> resolveEncoder(String name) {
+ Class<? extends Encoder> clazz = null;
+ try {
+ clazz = lookupEncoder(PACKAGE_CONTAINIG_ENCODERS+name);
+ } catch (ClassNotFoundException e) {
+ try {
+ clazz = lookupEncoder(name);
+ } catch (ClassNotFoundException cnfe) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown encoder: "+name +" ["+registry.keySet()+"]" );
+ }
+ }
+ catch (ClassCastException e) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Not an encoder: "+name +" ["+registry.keySet()+"]" );
+ }
+ return clazz;
+ }
+
+ private Class<? extends Encoder> lookupEncoder(String name)
+ throws ClassNotFoundException {
+ Class<? extends Encoder> clazz = Class.forName(name).asSubclass(Encoder.class);
+ registry.put( name.toUpperCase(Locale.ENGLISH), clazz );
+ return clazz;
+ }
+
public PhoneticFilter create(TokenStream input) {
return new PhoneticFilter(input,encoder,inject);
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java?rev=1195082&r1=1195081&r2=1195082&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java Sun Oct 30 01:00:06 2011
@@ -32,11 +32,35 @@ import org.apache.lucene.analysis.Tokeni
*/
public class TestPhoneticFilterFactory extends BaseTokenTestCase {
+ private static final int REPEATS = 100000;
+
+ /**
+ * Case: default
+ */
public void testFactory()
{
Map<String,String> args = new HashMap<String, String>();
PhoneticFilterFactory ff = new PhoneticFilterFactory();
+
+ args.put( PhoneticFilterFactory.ENCODER, "Metaphone" );
+ ff.init( args );
+ assertTrue( ff.encoder instanceof Metaphone );
+ assertTrue( ff.inject ); // default
+
+ args.put( PhoneticFilterFactory.INJECT, "false" );
+ ff.init( args );
+ assertFalse( ff.inject );
+ }
+
+ /**
+ * Case: Failures and Exceptions
+ */
+ public void testFactoryCaseFailure()
+ {
+ Map<String,String> args = new HashMap<String, String>();
+
+ PhoneticFilterFactory ff = new PhoneticFilterFactory();
try {
ff.init( args );
fail( "missing encoder parameter" );
@@ -48,15 +72,27 @@ public class TestPhoneticFilterFactory e
fail( "unknown encoder parameter" );
}
catch( Exception ex ) {}
+ args.put( PhoneticFilterFactory.ENCODER, "org.apache.commons.codec.language.NonExistence" );
+ try {
+ ff.init( args );
+ fail( "unknown encoder parameter" );
+ }
+ catch( Exception ex ) {}
+ }
+
+ /**
+ * Case: Reflection
+ */
+ public void testFactoryCaseReflection()
+ {
+ Map<String,String> args = new HashMap<String, String>();
- args.put( PhoneticFilterFactory.ENCODER, "Metaphone" );
+ PhoneticFilterFactory ff = new PhoneticFilterFactory();
+
+ args.put( PhoneticFilterFactory.ENCODER, "org.apache.commons.codec.language.Metaphone" );
ff.init( args );
assertTrue( ff.encoder instanceof Metaphone );
assertTrue( ff.inject ); // default
-
- args.put( PhoneticFilterFactory.INJECT, "false" );
- ff.init( args );
- assertFalse( ff.inject );
}
public void testAlgorithms() throws Exception {
@@ -85,6 +121,12 @@ public class TestPhoneticFilterFactory e
"TTA1111111", "Datha", "KLN1111111", "Carlene" });
assertAlgorithm("Caverphone", "false", "Darda Karleen Datha Carlene",
new String[] { "TTA1111111", "KLN1111111", "TTA1111111", "KLN1111111" });
+
+ assertAlgorithm("ColognePhonetic", "true", "Meier Schmitt Meir Schmidt",
+ new String[] { "67", "Meier", "862", "Schmitt",
+ "67", "Meir", "862", "Schmidt" });
+ assertAlgorithm("ColognePhonetic", "false", "Meier Schmitt Meir Schmidt",
+ new String[] { "67", "862", "67", "862" });
}
static void assertAlgorithm(String algName, String inject, String input,
@@ -98,4 +140,25 @@ public class TestPhoneticFilterFactory e
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, expected);
}
+
+ public void testSpeed() throws Exception {
+ checkSpeedEncoding("Metaphone", "easgasg", "ESKS");
+ checkSpeedEncoding("DoubleMetaphone", "easgasg", "ASKS");
+ checkSpeedEncoding("Soundex", "easgasg", "E220");
+ checkSpeedEncoding("RefinedSoundex", "easgasg", "E034034");
+ checkSpeedEncoding("Caverphone", "Carlene", "KLN1111111");
+ checkSpeedEncoding("ColognePhonetic", "Schmitt", "862");
+ }
+
+ private void checkSpeedEncoding(String encoder, String toBeEncoded, String estimated) throws Exception {
+ long start = System.currentTimeMillis();
+ for ( int i=0; i<REPEATS; i++) {
+ assertAlgorithm(encoder, "false", toBeEncoded,
+ new String[] { estimated });
+ }
+ long duration = System.currentTimeMillis()-start;
+ if (VERBOSE)
+ System.out.println(encoder + " encodings per msec: "+(REPEATS/duration));
+ }
+
}
Added: lucene/dev/trunk/solr/lib/commons-codec-1.5.jar
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/lib/commons-codec-1.5.jar?rev=1195082&view=auto
==============================================================================
Binary file - no diff available.