You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2012/02/25 15:14:00 UTC

svn commit: r1293614 - in /lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima: BaseUIMATokenizer.java UIMAAnnotationsTokenizer.java UIMATypeAwareAnnotationsTokenizer.java

Author: tommaso
Date: Sat Feb 25 14:14:00 2012
New Revision: 1293614

URL: http://svn.apache.org/viewvc?rev=1293614&view=rev
Log:
[LUCENE-3731] - refactored analyzeText method to initializeIterator and made it abstract inside BaseUIMATokenizer 

Modified:
    lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
    lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
    lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java

Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java Sat Feb 25 14:14:00 2012
@@ -51,18 +51,25 @@ public abstract class BaseUIMATokenizer 
 
   /**
    * analyzes the tokenizer input using the given analysis engine
-   * 
+   * <p/>
    * {@link #cas} will be filled with  extracted metadata (UIMA annotations, feature structures)
    *
    * @throws AnalysisEngineProcessException
    * @throws IOException
    */
-  protected void analyzeInput() throws AnalysisEngineProcessException,IOException {
+  protected void analyzeInput() throws AnalysisEngineProcessException, IOException {
     cas.reset();
     cas.setDocumentText(toString(input));
     ae.process(cas);
   }
 
+  /**
+   * initialize the FSIterator which is used to build tokens at each incrementToken() method call
+   *
+   * @throws IOException
+   */
+  protected abstract void initializeIterator() throws IOException;
+
   private String toString(Reader reader) throws IOException {
     StringBuilder stringBuilder = new StringBuilder();
     int ch;
@@ -82,6 +89,6 @@ public abstract class BaseUIMATokenizer 
   public void end() throws IOException {
     iterator = null;
   }
-  
-  
+
+
 }

Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java Sat Feb 25 14:14:00 2012
@@ -37,7 +37,7 @@ public final class UIMAAnnotationsTokeni
   private final OffsetAttribute offsetAttr;
 
   private final String tokenTypeString;
-  
+
   private int finalOffset = 0;
 
   public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Reader input) {
@@ -47,8 +47,12 @@ public final class UIMAAnnotationsTokeni
     this.offsetAttr = addAttribute(OffsetAttribute.class);
   }
 
-  private void analyzeText() throws IOException, AnalysisEngineProcessException {
-    analyzeInput();
+  protected void initializeIterator() throws IOException {
+    try {
+      analyzeInput();
+    } catch (AnalysisEngineProcessException e) {
+      throw new IOException(e);
+    }
     finalOffset = correctOffset(cas.getDocumentText().length());
     Type tokenType = cas.getTypeSystem().getType(tokenTypeString);
     iterator = cas.getAnnotationIndex(tokenType).iterator();
@@ -57,11 +61,7 @@ public final class UIMAAnnotationsTokeni
   @Override
   public boolean incrementToken() throws IOException {
     if (iterator == null) {
-      try {
-        analyzeText();
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
+      initializeIterator();
     }
     if (iterator.hasNext()) {
       clearAttributes();

Modified: lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java?rev=1293614&r1=1293613&r2=1293614&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java Sat Feb 25 14:14:00 2012
@@ -47,7 +47,7 @@ public final class UIMATypeAwareAnnotati
   private final String typeAttributeFeaturePath;
 
   private FeaturePath featurePath;
-  
+
   private int finalOffset = 0;
 
   public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath, Reader input) {
@@ -59,23 +59,29 @@ public final class UIMATypeAwareAnnotati
     this.typeAttributeFeaturePath = typeAttributeFeaturePath;
   }
 
-  private void analyzeText() throws IOException, AnalysisEngineProcessException, CASException {
-    analyzeInput();
+  protected void initializeIterator() throws IOException {
+    try {
+      analyzeInput();
+    } catch (AnalysisEngineProcessException e) {
+      throw new IOException(e);
+    }
+    featurePath = cas.createFeaturePath();
+    try {
+      featurePath.initialize(typeAttributeFeaturePath);
+    } catch (CASException e) {
+      featurePath = null;
+      throw new IOException(e);
+    }
     finalOffset = correctOffset(cas.getDocumentText().length());
     Type tokenType = cas.getTypeSystem().getType(tokenTypeString);
     iterator = cas.getAnnotationIndex(tokenType).iterator();
-    featurePath = cas.createFeaturePath();
-    featurePath.initialize(typeAttributeFeaturePath);
+
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (iterator == null) {
-      try {
-        analyzeText();
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
+      initializeIterator();
     }
     if (iterator.hasNext()) {
       clearAttributes();