You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2012/02/25 15:14:00 UTC
svn commit: r1293614 - in
/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima:
BaseUIMATokenizer.java UIMAAnnotationsTokenizer.java
UIMATypeAwareAnnotationsTokenizer.java
Author: tommaso
Date: Sat Feb 25 14:14:00 2012
New Revision: 1293614
URL: http://svn.apache.org/viewvc?rev=1293614&view=rev
Log:
[LUCENE-3731] - refactored analyzeText method to initializeIterator and made it abstract inside BaseUIMATokenizer
Modified:
lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java Sat Feb 25 14:14:00 2012
@@ -51,18 +51,25 @@ public abstract class BaseUIMATokenizer
/**
* analyzes the tokenizer input using the given analysis engine
- *
+ * <p/>
* {@link #cas} will be filled with extracted metadata (UIMA annotations, feature structures)
*
* @throws AnalysisEngineProcessException
* @throws IOException
*/
- protected void analyzeInput() throws AnalysisEngineProcessException,IOException {
+ protected void analyzeInput() throws AnalysisEngineProcessException, IOException {
cas.reset();
cas.setDocumentText(toString(input));
ae.process(cas);
}
+ /**
+ * initialize the FSIterator which is used to build tokens at each incrementToken() method call
+ *
+ * @throws IOException
+ */
+ protected abstract void initializeIterator() throws IOException;
+
private String toString(Reader reader) throws IOException {
StringBuilder stringBuilder = new StringBuilder();
int ch;
@@ -82,6 +89,6 @@ public abstract class BaseUIMATokenizer
public void end() throws IOException {
iterator = null;
}
-
-
+
+
}
Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java Sat Feb 25 14:14:00 2012
@@ -37,7 +37,7 @@ public final class UIMAAnnotationsTokeni
private final OffsetAttribute offsetAttr;
private final String tokenTypeString;
-
+
private int finalOffset = 0;
public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Reader input) {
@@ -47,8 +47,12 @@ public final class UIMAAnnotationsTokeni
this.offsetAttr = addAttribute(OffsetAttribute.class);
}
- private void analyzeText() throws IOException, AnalysisEngineProcessException {
- analyzeInput();
+ protected void initializeIterator() throws IOException {
+ try {
+ analyzeInput();
+ } catch (AnalysisEngineProcessException e) {
+ throw new IOException(e);
+ }
finalOffset = correctOffset(cas.getDocumentText().length());
Type tokenType = cas.getTypeSystem().getType(tokenTypeString);
iterator = cas.getAnnotationIndex(tokenType).iterator();
@@ -57,11 +61,7 @@ public final class UIMAAnnotationsTokeni
@Override
public boolean incrementToken() throws IOException {
if (iterator == null) {
- try {
- analyzeText();
- } catch (Exception e) {
- throw new IOException(e);
- }
+ initializeIterator();
}
if (iterator.hasNext()) {
clearAttributes();
Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java Sat Feb 25 14:14:00 2012
@@ -47,7 +47,7 @@ public final class UIMATypeAwareAnnotati
private final String typeAttributeFeaturePath;
private FeaturePath featurePath;
-
+
private int finalOffset = 0;
public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath, Reader input) {
@@ -59,23 +59,29 @@ public final class UIMATypeAwareAnnotati
this.typeAttributeFeaturePath = typeAttributeFeaturePath;
}
- private void analyzeText() throws IOException, AnalysisEngineProcessException, CASException {
- analyzeInput();
+ protected void initializeIterator() throws IOException {
+ try {
+ analyzeInput();
+ } catch (AnalysisEngineProcessException e) {
+ throw new IOException(e);
+ }
+ featurePath = cas.createFeaturePath();
+ try {
+ featurePath.initialize(typeAttributeFeaturePath);
+ } catch (CASException e) {
+ featurePath = null;
+ throw new IOException(e);
+ }
finalOffset = correctOffset(cas.getDocumentText().length());
Type tokenType = cas.getTypeSystem().getType(tokenTypeString);
iterator = cas.getAnnotationIndex(tokenType).iterator();
- featurePath = cas.createFeaturePath();
- featurePath.initialize(typeAttributeFeaturePath);
+
}
@Override
public boolean incrementToken() throws IOException {
if (iterator == null) {
- try {
- analyzeText();
- } catch (Exception e) {
- throw new IOException(e);
- }
+ initializeIterator();
}
if (iterator.hasNext()) {
clearAttributes();