You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/02/28 21:41:33 UTC
svn commit: r1573066 - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/
lucene/analysis/common/src/resources/META-INF/services/
lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/
solr/example/solr...
Author: rmuir
Date: Fri Feb 28 20:41:32 2014
New Revision: 1573066
URL: http://svn.apache.org/r1573066
Log:
LUCENE-5482: Improve default TurkishAnalyzer
Added:
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilter.java (with props)
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilterFactory.java (with props)
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilter.java (with props)
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilterFactory.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java
lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1573066&r1=1573065&r2=1573066&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Feb 28 20:41:32 2014
@@ -81,6 +81,9 @@ New Features
additions and updates (to change weight or payload of an existing
suggestion). (Mike McCandless)
+* LUCENE-5482: Improve default TurkishAnalyzer by adding apostrophe
+ handling suitable for Turkish. (Ahmet Arslan via Robert Muir)
+
API Changes
* LUCENE-5454: Add RandomAccessOrds, an optional extension of SortedSetDocValues
Added: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilter.java?rev=1573066&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilter.java (added)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilter.java Fri Feb 28 20:41:32 2014
@@ -0,0 +1,59 @@
+package org.apache.lucene.analysis.tr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+import java.io.IOException;
+
+/**
+ * Strips all characters after an apostrophe (including the apostrophe itself).
+ * <p>
+ * In Turkish, apostrophe is used to separate suffixes from proper names
+ * (continent, sea, river, lake, mountain, upland, proper names related to
+ * religion and mythology). This filter intended to be used before stem filters.
+ * For more information, see <a href="http://www.ipcsit.com/vol57/015-ICNI2012-M021.pdf">
+ * Role of Apostrophes in Turkish Information Retrieval</a>
+ * </p>
+ */
+public final class ApostropheFilter extends TokenFilter {
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public ApostropheFilter(TokenStream in) {
+ super(in);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (!input.incrementToken())
+ return false;
+
+ final char[] buffer = termAtt.buffer();
+ final int length = termAtt.length();
+
+ for (int i = 0; i < length; i++)
+ if (buffer[i] == '\'' || buffer[i] == '\u2019') {
+ termAtt.setLength(i);
+ return true;
+ }
+ return true;
+ }
+}
Added: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilterFactory.java?rev=1573066&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilterFactory.java (added)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/ApostropheFilterFactory.java Fri Feb 28 20:41:32 2014
@@ -0,0 +1,49 @@
+package org.apache.lucene.analysis.tr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+import java.util.Map;
+
+/**
+ * Factory for {@link ApostropheFilter}.
+ * <pre class="prettyprint">
+ * <fieldType name="text_tr_lower_apostrophes" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.StandardTokenizerFactory"/>
+ * <filter class="solr.ApostropheFilterFactory"/>
+ * <filter class="solr.TurkishLowerCaseFilterFactory"/>
+ * </analyzer>
+ * </fieldType></pre>
+ */
+public class ApostropheFilterFactory extends TokenFilterFactory {
+
+ public ApostropheFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameter(s): " + args);
+ }
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new ApostropheFilter(input);
+ }
+}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java?rev=1573066&r1=1573065&r2=1573066&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java Fri Feb 28 20:41:32 2014
@@ -122,6 +122,8 @@ public final class TurkishAnalyzer exten
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
+ if(matchVersion.onOrAfter(Version.LUCENE_48))
+ result = new ApostropheFilter(result);
result = new TurkishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
if(!stemExclusionSet.isEmpty())
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory?rev=1573066&r1=1573065&r2=1573066&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory Fri Feb 28 20:41:32 2014
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.lucene.analysis.tr.ApostropheFilterFactory
org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory
org.apache.lucene.analysis.ar.ArabicStemFilterFactory
org.apache.lucene.analysis.bg.BulgarianStemFilterFactory
Added: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilter.java?rev=1573066&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilter.java (added)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilter.java Fri Feb 28 20:41:32 2014
@@ -0,0 +1,32 @@
+package org.apache.lucene.analysis.tr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+
+
+public class TestApostropheFilter extends BaseTokenStreamTestCase {
+
+ public void testApostropheFilter() throws Exception {
+ TokenStream stream = whitespaceMockTokenizer("Türkiye'de 2003'te Van Gölü'nü gördüm");
+ stream = new TurkishLowerCaseFilter(stream);
+ stream = new ApostropheFilter(stream);
+ assertTokenStreamContents(stream, new String[]{"türkiye", "2003", "van", "gölü", "gördüm"});
+ }
+}
Added: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilterFactory.java?rev=1573066&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilterFactory.java (added)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestApostropheFilterFactory.java Fri Feb 28 20:41:32 2014
@@ -0,0 +1,55 @@
+package org.apache.lucene.analysis.tr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+import java.io.Reader;
+import java.io.StringReader;
+
+/**
+ * Simple tests to ensure the apostrophe filter factory is working.
+ */
+public class TestApostropheFilterFactory extends BaseTokenStreamFactoryTestCase {
+ /**
+ * Ensure the filter actually removes characters after an apostrophe.
+ */
+ public void testApostrophes() throws Exception {
+ Reader reader = new StringReader("Türkiye'de 2003'te Van Gölü'nü gördüm");
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer) stream).setReader(reader);
+ stream = tokenFilterFactory("Apostrophe").create(stream);
+ assertTokenStreamContents(stream, new String[]{"Türkiye", "2003", "Van", "Gölü", "gördüm"});
+ }
+
+ /**
+ * Test that bogus arguments result in exception
+ */
+ public void testBogusArguments() throws Exception {
+ try {
+ tokenFilterFactory("Apostrophe", "bogusArg", "bogusValue");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Unknown parameter(s):"));
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java?rev=1573066&r1=1573065&r2=1573066&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java Fri Feb 28 20:41:32 2014
@@ -38,6 +38,9 @@ public class TestTurkishAnalyzer extends
checkOneTerm(a, "aÄaç", "aÄaç");
// stopword
assertAnalyzesTo(a, "dolayı", new String[] {});
+ // apostrophes
+ checkOneTerm(a, "Kıbrıs'ta", "kıbrıs");
+ assertAnalyzesTo(a, "Van Gölü'ne", new String[]{"van", "göl"});
}
/** test use of exclusion set */
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml?rev=1573066&r1=1573065&r2=1573066&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml Fri Feb 28 20:41:32 2014
@@ -1124,6 +1124,7 @@
<fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ApostropheFilterFactory"/>
<filter class="solr.TurkishLowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_tr.txt" />
<filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>