You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2006/08/03 22:19:46 UTC
svn commit: r428512 - in /incubator/solr/trunk: CHANGES.txt
src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java
src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java
Author: yonik
Date: Thu Aug 3 13:19:45 2006
New Revision: 428512
URL: http://svn.apache.org/viewvc?rev=428512&view=rev
Log:
HyphenatedWordsFilter: SOLR-41
Added:
incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java (with props)
incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java (with props)
incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java (with props)
Modified:
incubator/solr/trunk/CHANGES.txt
Modified: incubator/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/CHANGES.txt?rev=428512&r1=428511&r2=428512&view=diff
==============================================================================
--- incubator/solr/trunk/CHANGES.txt (original)
+++ incubator/solr/trunk/CHANGES.txt Thu Aug 3 13:19:45 2006
@@ -40,6 +40,8 @@
The default operator remains "OR".
21. JAVA API: new version of SolrIndexSearcher.getDocListAndSet() which takes
flags (Greg Ludington via yonik, SOLR-39)
+22. A HyphenatedWordsFilter, a text analysis filter used during indexing to rejoin
+ words that were hyphenated and split by a newline. (Boris Vitez via yonik, SOLR-41)
Changes in runtime behavior
1. classes reorganized into different packages, package names changed to Apache
Added: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java?rev=428512&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java Thu Aug 3 13:19:45 2006
@@ -0,0 +1,94 @@
+package org.apache.solr.analysis;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.*;
+
+/**
+ * When the plain text is extracted from documents, we will often have many words hyphenated and broken into
+ * two lines. This is often the case with documents where narrow text columns are used, such as newsletters.
+ * In order to increase search efficiency, this filter puts hyphenated words broken into two lines back together.
+ * This filter should be used on indexing time only.
+ * Example field definition in schema.xml:
+ * <pre>
+ * <fieldtype name="text" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer type="index">
+ * <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ * <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ * <filter class="solr.StopFilterFactory" ignoreCase="true"/>
+ * <filter class="solr.HyphenatedWordsFilterFactory"/>
+ * <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+ * <filter class="solr.LowerCaseFilterFactory"/>
+ * <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ * </analyzer>
+ * <analyzer type="query">
+ * <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ * <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ * <filter class="solr.StopFilterFactory" ignoreCase="true"/>
+ * <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0"/>
+ * <filter class="solr.LowerCaseFilterFactory"/>
+ * <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ * </analyzer>
+ * </fieldtype>
+ *
+ * @author Boris Vitez
+ */
+public final class HyphenatedWordsFilter extends TokenFilter {
+
+ public HyphenatedWordsFilter(TokenStream in) {
+ super(in);
+ }
+
+ /**
+ * @inheritDoc
+ * @see org.apache.lucene.analysis.TokenStream#next()
+ */
+ public final Token next() throws IOException {
+ StringBuffer termText = new StringBuffer(25);
+ int startOffset = -1, firstPositionIncrement = -1, wordsMerged = 0;
+ Token lastToken = null;
+ for (Token token = input.next(); token != null; token = input.next()) {
+ termText.append(token.termText());
+ //current token ends with hyphen -> grab the next token and glue them together
+ if (termText.charAt(termText.length() - 1) == '-') {
+ wordsMerged++;
+ //remove the hyphen
+ termText.setLength(termText.length()-1);
+ if (startOffset == -1) {
+ startOffset = token.startOffset();
+ firstPositionIncrement = token.getPositionIncrement();
+ }
+ lastToken = token;
+ } else {
+ //shortcut returns token
+ if (wordsMerged == 0)
+ return token;
+ Token mergedToken = new Token(termText.toString(), startOffset, token.endOffset(), token.type());
+ mergedToken.setPositionIncrement(firstPositionIncrement);
+ return mergedToken;
+ }
+ }
+ //last token ending with hyphen? - we know that we have only one token in
+ //this situation, so we can safely return firstToken
+ if (startOffset != -1)
+ return lastToken;
+ else
+ return null; //end of token stream
+ }
+}
Propchange: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilter.java
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java?rev=428512&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java Thu Aug 3 13:19:45 2006
@@ -0,0 +1,30 @@
+package org.apache.solr.analysis;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.solr.analysis.BaseTokenFilterFactory;
+
+/**
+ * Factory for HyphenatedWordsFilter
+ * @author Boris Vitez
+ */
+public class HyphenatedWordsFilterFactory extends BaseTokenFilterFactory {
+ public TokenStream create(TokenStream input) {
+ return new HyphenatedWordsFilter(input);
+ }
+}
Propchange: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/solr/trunk/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java?rev=428512&view=auto
==============================================================================
--- incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java (added)
+++ incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java Thu Aug 3 13:19:45 2006
@@ -0,0 +1,38 @@
+package org.apache.solr.analysis;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+
+/**
+ * HyphenatedWordsFilter test
+ */
+public class TestHyphenatedWordsFilter extends TestCase {
+ public void testHyphenatedWords() throws Exception {
+ String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on";
+ String outputAfterHyphenatedWordsFilter = "ecological develop comprehensive-hands-on";
+ // first test
+ TokenStream ts = new WhitespaceTokenizer(new StringReader(input));
+ ts = new HyphenatedWordsFilter(ts);
+ String actual = tsToString(ts);
+ assertEquals("Testing HyphenatedWordsFilter",
+ outputAfterHyphenatedWordsFilter, actual);
+ }
+
+ public static String tsToString(TokenStream in) throws IOException {
+ StringBuffer out = new StringBuffer();
+ Token t = in.next();
+ if (null != t)
+ out.append(t.termText());
+
+ for (t = in.next(); null != t; t = in.next()) {
+ out.append(" ").append(t.termText());
+ }
+ return out.toString();
+ }
+}
Propchange: incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/solr/trunk/src/test/org/apache/solr/analysis/TestHyphenatedWordsFilter.java
------------------------------------------------------------------------------
svn:executable = *