You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/06/23 21:09:02 UTC
svn commit: r787792 - in
/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene:
analysis/CharReader.java analysis/Tokenizer.java
analysis/standard/StandardTokenizer.java search/CharStream.java
Author: mikemccand
Date: Tue Jun 23 19:09:01 2009
New Revision: 787792
URL: http://svn.apache.org/viewvc?rev=787792&view=rev
Log:
LUCENE-1466: added chainable CharFilter stage before Tokenizer to allow mapping of characters before tokenization
Added:
lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java (with props)
lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java (with props)
Modified:
lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Added: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java?rev=787792&view=auto
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java (added)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java Tue Jun 23 19:09:01 2009
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * CharReader is a Reader wrapper. It reads chars from Reader and outputs CharStream.
+ *
+ * @version $Id$
+ *
+ */
+public final class CharReader extends CharStream {
+
+ protected Reader input;
+
+ public static CharStream get(Reader input) {
+ return input instanceof CharStream ?
+ (CharStream)input : new CharReader(input);
+ }
+
+ private CharReader(Reader in) {
+ input = in;
+ }
+
+ public int correctOffset(int currentOff) {
+ return currentOff;
+ }
+
+ public void close() throws IOException {
+ input.close();
+ }
+
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ return input.read(cbuf, off, len);
+ }
+}
Propchange: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/CharReader.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=787792&r1=787791&r2=787792&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java Tue Jun 23 19:09:01 2009
@@ -45,14 +45,14 @@
public abstract class Tokenizer extends TokenStream {
/** The text source for this Tokenizer. */
- protected Reader input;
+ protected CharStream input;
/** Construct a tokenizer with null input. */
protected Tokenizer() {}
/** Construct a token stream processing the given input. */
protected Tokenizer(Reader input) {
- this.input = input;
+ this.input = CharReader.get(input);
}
/** By default, closes the input Reader. */
@@ -64,7 +64,7 @@
* analyzer (in its reusableTokenStream method) will use
* this to re-use a previously created tokenizer. */
public void reset(Reader input) throws IOException {
- this.input = input;
+ this.input = CharReader.get(input);
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=787792&r1=787791&r2=787792&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Tue Jun 23 19:09:01 2009
@@ -26,6 +26,8 @@
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.CharReader;
+import org.apache.lucene.analysis.CharStream;
/** A grammar-based tokenizer constructed with JFlex
*
@@ -91,7 +93,7 @@
private boolean replaceInvalidAcronym;
void setInput(Reader reader) {
- this.input = reader;
+ this.input = CharReader.get(reader);
}
private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
@@ -126,7 +128,7 @@
*/
public StandardTokenizer(Reader input, boolean replaceInvalidAcronym) {
this.replaceInvalidAcronym = replaceInvalidAcronym;
- this.input = input;
+ setInput(input);
this.scanner = new StandardTokenizerImpl(input);
termAtt = (TermAttribute) addAttribute(TermAttribute.class);
offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
@@ -240,7 +242,7 @@
}
public void reset(Reader reader) throws IOException {
- input = reader;
+ setInput(reader);
reset();
}
Added: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java?rev=787792&view=auto
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java (added)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java Tue Jun 23 19:09:01 2009
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.Reader;
+
+/**
+ * CharStream adds <a href="#correctOffset(int)">correctOffset</a> functionality over Reader.
+ *
+ * @version $Id$
+ *
+ */
+public abstract class CharStream extends Reader {
+
+ /**
+ * called by CharFilter(s) and Tokenizer to correct token offset.
+ *
+ * @param currentOff current offset
+ * @return corrected token offset
+ */
+ public abstract int correctOffset( int currentOff );
+}
Propchange: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/CharStream.java
------------------------------------------------------------------------------
svn:eol-style = native