You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ctakes.apache.org by vj...@apache.org on 2014/01/03 19:45:45 UTC
svn commit: r1555196 - in
/ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api:
CharacterOffsetToLineTokenConverterCtakesImpl.java
SingleDocumentProcessorCtakes.java
Author: vjapache
Date: Fri Jan 3 18:45:44 2014
New Revision: 1555196
URL: http://svn.apache.org/r1555196
Log:
fix character offset calculation for sentences that contain newlines
Modified:
ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/CharacterOffsetToLineTokenConverterCtakesImpl.java
ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/SingleDocumentProcessorCtakes.java
Modified: ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/CharacterOffsetToLineTokenConverterCtakesImpl.java
URL: http://svn.apache.org/viewvc/ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/CharacterOffsetToLineTokenConverterCtakesImpl.java?rev=1555196&r1=1555195&r2=1555196&view=diff
==============================================================================
--- ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/CharacterOffsetToLineTokenConverterCtakesImpl.java (original)
+++ ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/CharacterOffsetToLineTokenConverterCtakesImpl.java Fri Jan 3 18:45:44 2014
@@ -19,11 +19,13 @@
package org.apache.ctakes.assertion.medfacts.i2b2.api;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
-import org.apache.log4j.Logger;
+import org.apache.log4j.Logger;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationIndex;
@@ -43,6 +45,7 @@ public class CharacterOffsetToLineTokenC
protected TreeMap<Integer, Sentence> beginTreeMap;
protected TreeSet<Integer> tokenBeginEndTreeSet;
+ protected Map<Sentence, List<BaseToken>> sentenceToTokenNumberMap;
public CharacterOffsetToLineTokenConverterCtakesImpl()
{
@@ -54,6 +57,7 @@ public class CharacterOffsetToLineTokenC
this.jcas = jcas;
buildSentenceBoundaryMap();
buildTokenBoundaryMap();
+ buildSentenceToTokenNumberMap();
}
public void buildSentenceBoundaryMap()
@@ -88,6 +92,20 @@ public class CharacterOffsetToLineTokenC
}
}
+ protected void buildSentenceToTokenNumberMap() {
+ sentenceToTokenNumberMap = new HashMap<Sentence, List<BaseToken>>();
+ for (Sentence s : beginTreeMap.values()) {
+ FSIterator<Annotation> tokensInSentenceIterator = jcas
+ .getAnnotationIndex(BaseToken.type).subiterator(s);
+ List<BaseToken> btList = new ArrayList<BaseToken>();
+ BaseToken bt = null;
+ while ((bt = this.getNextNonEOLToken(tokensInSentenceIterator)) != null) {
+ btList.add(bt);
+ }
+ sentenceToTokenNumberMap.put(s, btList);
+ }
+ }
+
public Sentence findPreviousOrCurrentSentence(int characterOffset)
{
Integer floorKey = beginTreeMap.floorKey(characterOffset);
@@ -173,36 +191,29 @@ public class CharacterOffsetToLineTokenC
int lineNumber = sentence.getSentenceNumber() + 1;
- FSIterator<Annotation> tokensInSentenceIterator =
- jcas.getAnnotationIndex(baseTokenTypeId).subiterator(sentence);
-
- if (!tokensInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation firstTokenAnnotation = tokensInSentenceIterator.next();
- BaseToken firstToken = (BaseToken)firstTokenAnnotation;
- int firstTokenInSentenceNumber = firstToken.getTokenNumber();
-
-
- FSIterator<Annotation> beginTokenInSentenceIterator =
- constraintConstructorFindContainedBy.createFilteredIterator(
- characterOffset, characterOffset, baseTokenType);
-
- if (!beginTokenInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!! (character offset request = " + characterOffset);
- }
- Annotation beginTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken beginToken = (BaseToken)beginTokenAnnotation;
- int beginTokenNumber = beginToken.getTokenNumber();
- int beginTokenWordNumber = beginTokenNumber - firstTokenInSentenceNumber;
-
- LineAndTokenPosition b = new LineAndTokenPosition();
- b.setLine(lineNumber);
- b.setTokenOffset(beginTokenWordNumber);
-
- return b;
+ FSIterator<Annotation> beginTokenInSentenceIterator = constraintConstructorFindContainedBy
+ .createFilteredIterator(characterOffset, characterOffset,
+ baseTokenType);
+ BaseToken beginToken = this
+ .getNextNonEOLToken(beginTokenInSentenceIterator);
+ int beginTokenWordNumber = this.sentenceToTokenNumberMap.get(sentence)
+ .indexOf(beginToken);
+ LineAndTokenPosition b = new LineAndTokenPosition();
+ b.setLine(lineNumber);
+ b.setTokenOffset(beginTokenWordNumber);
+
+ return b;
+ }
+
+ public BaseToken getNextNonEOLToken(
+ FSIterator<Annotation> tokensInSentenceIterator) {
+ while (tokensInSentenceIterator.hasNext()) {
+ BaseToken bt = (BaseToken) tokensInSentenceIterator.next();
+ if (!(bt instanceof NewlineToken)) {
+ return bt;
+ }
+ }
+ return null;
}
public List<LineAndTokenPosition> calculateBeginAndEndOfConcept
@@ -214,79 +225,82 @@ public class CharacterOffsetToLineTokenC
public List<LineAndTokenPosition> calculateBeginAndEndOfConcept(
int problemBegin, int problemEnd)
{
- //int externalId = problem.getExternalId();
- //int sentenceTypeId = Sentence.type;
- int baseTokenTypeId = BaseToken.type;
- //jcas.getAnnotationIndex(sentenceTypeId);
-
- ConstraintConstructorFindContainedBy constraintConstructorFindContainedBy = new ConstraintConstructorFindContainedBy(jcas);
- ConstraintConstructorFindContainedWithin constraintConstructorFindContainedWithin = new ConstraintConstructorFindContainedWithin(jcas);
-
- //AnnotationIndex<Annotation> sentenceAnnotationIndex = jcas.getAnnotationIndex(sentenceTypeId);
- Type sentenceType = jcas.getTypeSystem().getType(Sentence.class.getName());
- Type baseTokenType = jcas.getTypeSystem().getType(BaseToken.class.getName());
- ///
- FSIterator<Annotation> filteredIterator =
- constraintConstructorFindContainedBy.createFilteredIterator(
- problemBegin, problemEnd, sentenceType);
- ///
- if (!filteredIterator.hasNext())
- {
- throw new RuntimeException("Surrounding sentence annotation not found!!");
- }
- Annotation sentenceAnnotation = filteredIterator.next();
- Sentence sentence = (Sentence)sentenceAnnotation;
- int lineNumber = sentence.getSentenceNumber() + 1;
-
-
- FSIterator<Annotation> tokensInSentenceIterator =
- jcas.getAnnotationIndex(baseTokenTypeId).subiterator(sentence);
-
- if (!tokensInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation firstTokenAnnotation = tokensInSentenceIterator.next();
- BaseToken firstToken = (BaseToken)firstTokenAnnotation;
- int firstTokenInSentenceNumber = firstToken.getTokenNumber();
-
-
- FSIterator<Annotation> beginTokenInSentenceIterator =
- constraintConstructorFindContainedWithin.createFilteredIterator(
- problemBegin, problemEnd, baseTokenType);
-
- if (!beginTokenInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation beginTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken beginToken = (BaseToken)beginTokenAnnotation;
- int beginTokenNumber = beginToken.getTokenNumber();
- int beginTokenWordNumber = beginTokenNumber - firstTokenInSentenceNumber;
-
-
- beginTokenInSentenceIterator.moveToLast();
- if (!beginTokenInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation endTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken endToken = (BaseToken)endTokenAnnotation;
- int endTokenNumber = endToken.getTokenNumber();
- int endTokenWordNumber = endTokenNumber - firstTokenInSentenceNumber;
-
-
- ArrayList<LineAndTokenPosition> list = new ArrayList<LineAndTokenPosition>();
- LineAndTokenPosition b = new LineAndTokenPosition();
- b.setLine(lineNumber);
- b.setTokenOffset(beginTokenWordNumber);
- list.add(b);
- LineAndTokenPosition e = new LineAndTokenPosition();
- e.setLine(lineNumber);
- e.setTokenOffset(endTokenWordNumber);
- System.out.println("Adding lineTokenEnding " + lineNumber + " offset = " + endTokenWordNumber);
- list.add(e);
- return list;
+ // int externalId = problem.getExternalId();
+ // int sentenceTypeId = Sentence.type;
+ int baseTokenTypeId = BaseToken.type;
+ // jcas.getAnnotationIndex(sentenceTypeId);
+
+ ConstraintConstructorFindContainedBy constraintConstructorFindContainedBy = new ConstraintConstructorFindContainedBy(
+ jcas);
+ ConstraintConstructorFindContainedWithin constraintConstructorFindContainedWithin = new ConstraintConstructorFindContainedWithin(
+ jcas);
+
+ // AnnotationIndex<Annotation> sentenceAnnotationIndex =
+ // jcas.getAnnotationIndex(sentenceTypeId);
+ Type sentenceType = jcas.getTypeSystem().getType(
+ Sentence.class.getName());
+ Type baseTokenType = jcas.getTypeSystem().getType(
+ BaseToken.class.getName());
+ // /
+ FSIterator<Annotation> filteredIterator = constraintConstructorFindContainedBy
+ .createFilteredIterator(problemBegin, problemEnd, sentenceType);
+ // /
+ if (!filteredIterator.hasNext()) {
+ throw new RuntimeException(
+ "Surrounding sentence annotation not found!!");
+ }
+ Annotation sentenceAnnotation = filteredIterator.next();
+ Sentence sentence = (Sentence) sentenceAnnotation;
+ int lineNumber = sentence.getSentenceNumber() + 1;
+
+ // FSIterator<Annotation> tokensInSentenceIterator = jcas
+ // .getAnnotationIndex(baseTokenTypeId).subiterator(sentence);
+ //
+ // if (!tokensInSentenceIterator.hasNext()) {
+ // throw new RuntimeException("First token in sentence not found!!");
+ // }
+ // Annotation firstTokenAnnotation = tokensInSentenceIterator.next();
+ // BaseToken firstToken = (BaseToken) firstTokenAnnotation;
+ // int firstTokenInSentenceNumber = firstToken.getTokenNumber();
+
+ FSIterator<Annotation> beginTokenInSentenceIterator = constraintConstructorFindContainedWithin
+ .createFilteredIterator(problemBegin, problemEnd, baseTokenType);
+
+ // if (!beginTokenInSentenceIterator.hasNext()) {
+ // throw new RuntimeException("First token in sentence not found!!");
+ // }
+ // Annotation beginTokenAnnotation =
+ // beginTokenInSentenceIterator.next();
+ // BaseToken beginToken = (BaseToken) beginTokenAnnotation;
+ // int beginTokenNumber = beginToken.getTokenNumber();
+ // int beginTokenWordNumber = beginTokenNumber
+ // - firstTokenInSentenceNumber;
+ BaseToken beginToken = this
+ .getNextNonEOLToken(beginTokenInSentenceIterator);
+ int beginTokenWordNumber = this.sentenceToTokenNumberMap.get(sentence)
+ .indexOf(beginToken);
+
+ beginTokenInSentenceIterator.moveToLast();
+ if (!beginTokenInSentenceIterator.hasNext()) {
+ throw new RuntimeException("First token in sentence not found!!");
+ }
+ Annotation endTokenAnnotation = beginTokenInSentenceIterator.next();
+ BaseToken endToken = (BaseToken) endTokenAnnotation;
+ // int endTokenNumber = endToken.getTokenNumber();
+ // int endTokenWordNumber = endTokenNumber - firstTokenInSentenceNumber;
+ int endTokenWordNumber = this.sentenceToTokenNumberMap.get(sentence)
+ .indexOf(endToken);
+
+ ArrayList<LineAndTokenPosition> list = new ArrayList<LineAndTokenPosition>();
+ LineAndTokenPosition b = new LineAndTokenPosition();
+ b.setLine(lineNumber);
+ b.setTokenOffset(beginTokenWordNumber);
+ list.add(b);
+ LineAndTokenPosition e = new LineAndTokenPosition();
+ e.setLine(lineNumber);
+ e.setTokenOffset(endTokenWordNumber);
+ list.add(e);
+ return list;
}
}
Modified: ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/SingleDocumentProcessorCtakes.java
URL: http://svn.apache.org/viewvc/ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/SingleDocumentProcessorCtakes.java?rev=1555196&r1=1555195&r2=1555196&view=diff
==============================================================================
--- ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/SingleDocumentProcessorCtakes.java (original)
+++ ctakes/branches/ytex/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/medfacts/i2b2/api/SingleDocumentProcessorCtakes.java Fri Jan 3 18:45:44 2014
@@ -156,214 +156,31 @@ public class SingleDocumentProcessorCtak
return arrayOfLines;
}
+ /**
+ * delegate to converter to determine offset.
+ */
+ @Override
public LineAndTokenPosition convertCharacterOffsetToLineToken(int characterOffset)
{
- int baseTokenTypeId = BaseToken.type;
-
- ConstraintConstructorFindContainedBy constraintConstructorFindContainedBy = new ConstraintConstructorFindContainedBy(jcas);
- ConstraintConstructorFindContainedWithin constraintConstructorFindContainedWithin = new ConstraintConstructorFindContainedWithin(jcas);
-
- Type sentenceType = jcas.getTypeSystem().getType(Sentence.class.getName());
- Type baseTokenType = jcas.getTypeSystem().getType(BaseToken.class.getName());
-
- FSIterator<Annotation> filteredIterator =
- constraintConstructorFindContainedBy.createFilteredIterator(
- characterOffset, characterOffset, sentenceType);
-
- if (!filteredIterator.hasNext())
- {
- throw new RuntimeException("Surrounding sentence annotation not found!!");
- }
- Annotation sentenceAnnotation = filteredIterator.next();
- Sentence sentence = (Sentence)sentenceAnnotation;
- int lineNumber = sentence.getSentenceNumber() + 1;
-
-
- FSIterator<Annotation> tokensInSentenceIterator =
- jcas.getAnnotationIndex(baseTokenTypeId).subiterator(sentence);
-
- if (!tokensInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation firstTokenAnnotation = tokensInSentenceIterator.next();
- BaseToken firstToken = (BaseToken)firstTokenAnnotation;
- int firstTokenInSentenceNumber = firstToken.getTokenNumber();
-
-
- FSIterator<Annotation> beginTokenInSentenceIterator =
- constraintConstructorFindContainedBy.createFilteredIterator(
- characterOffset, characterOffset, baseTokenType);
-
- if (!beginTokenInSentenceIterator.hasNext())
- {
- throw new RuntimeException("First token in sentence not found!!");
- }
- Annotation beginTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken beginToken = (BaseToken)beginTokenAnnotation;
- int beginTokenNumber = beginToken.getTokenNumber();
- int beginTokenWordNumber = beginTokenNumber - firstTokenInSentenceNumber;
-
- LineAndTokenPosition b = new LineAndTokenPosition();
- b.setLine(lineNumber);
- b.setTokenOffset(beginTokenWordNumber);
-
- return b;
+ return converter2.convert(characterOffset);
}
+ @Override
public List<LineAndTokenPosition> calculateBeginAndEndOfConcept
(ApiConcept problem)
{
return calculateBeginAndEndOfConcept(problem.getBegin(), problem.getEnd());
}
+ /**
+ * delegate to converter to determine offset.
+ */
public List<LineAndTokenPosition> calculateBeginAndEndOfConcept(
int problemBegin, int problemEnd)
{
- //int externalId = problem.getExternalId();
- //int sentenceTypeId = Sentence.type;
- int baseTokenTypeId = BaseToken.type;
- //jcas.getAnnotationIndex(sentenceTypeId);
-
- ConstraintConstructorFindContainedBy constraintConstructorFindContainedBy = new ConstraintConstructorFindContainedBy(jcas);
- ConstraintConstructorFindContainedWithin constraintConstructorFindContainedWithin = new ConstraintConstructorFindContainedWithin(jcas);
-
- //AnnotationIndex<Annotation> sentenceAnnotationIndex = jcas.getAnnotationIndex(sentenceTypeId);
- Type sentenceType = jcas.getTypeSystem().getType(Sentence.class.getName());
- Type baseTokenType = jcas.getTypeSystem().getType(BaseToken.class.getName());
- ///
- FSIterator<Annotation> filteredIterator =
- constraintConstructorFindContainedBy.createFilteredIterator(
- problemBegin, problemEnd, sentenceType);
- ///
- if (!filteredIterator.hasNext())
- {
- return null;
- //throw new RuntimeException("Surrounding sentence annotation not found!!");
- }
- Annotation sentenceAnnotation = filteredIterator.next();
- Sentence sentence = (Sentence)sentenceAnnotation;
- int lineNumber = sentence.getSentenceNumber() + 1;
-
-
- FSIterator<Annotation> tokensInSentenceIterator =
- jcas.getAnnotationIndex(baseTokenTypeId).subiterator(sentence);
-
- if (!tokensInSentenceIterator.hasNext())
- {
- String message = "First token in sentence not found [#1]!!";
- Exception e = new RuntimeException(message);
- logger.error(message,e);
- return null;
- }
- Annotation firstTokenAnnotation = tokensInSentenceIterator.next();
- BaseToken firstToken = (BaseToken)firstTokenAnnotation;
- int firstTokenInSentenceNumber = firstToken.getTokenNumber();
-
-
- FSIterator<Annotation> beginTokenInSentenceIterator =
- constraintConstructorFindContainedWithin.createFilteredIterator(
- problemBegin, problemEnd, baseTokenType);
-
- if (!beginTokenInSentenceIterator.hasNext())
- {
- String message = "First token in sentence not found [#2]!!";
- Exception e = new RuntimeException(message);
- logger.error(message, e);
- return null;
- }
- Annotation beginTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken beginToken = (BaseToken)beginTokenAnnotation;
- int beginTokenNumber = beginToken.getTokenNumber();
- int beginTokenWordNumber = beginTokenNumber - firstTokenInSentenceNumber;
-
-
- beginTokenInSentenceIterator.moveToLast();
- if (!beginTokenInSentenceIterator.hasNext())
- {
- String message = "First token in sentence not found [#3]!!";
- Exception e = new RuntimeException(message);
- logger.error(message, e);
- return null;
- }
- Annotation endTokenAnnotation = beginTokenInSentenceIterator.next();
- BaseToken endToken = (BaseToken)endTokenAnnotation;
- int endTokenNumber = endToken.getTokenNumber();
- int endTokenWordNumber = endTokenNumber - firstTokenInSentenceNumber;
-
-
- ArrayList<LineAndTokenPosition> list = new ArrayList<LineAndTokenPosition>();
- LineAndTokenPosition b = new LineAndTokenPosition();
- b.setLine(lineNumber);
- b.setTokenOffset(beginTokenWordNumber);
- list.add(b);
- LineAndTokenPosition e = new LineAndTokenPosition();
- e.setLine(lineNumber);
- e.setTokenOffset(endTokenWordNumber);
- list.add(e);
- return list;
+ return ((CharacterOffsetToLineTokenConverterCtakesImpl) this.converter2)
+ .calculateBeginAndEndOfConcept(problemBegin, problemEnd);
}
-// /**
-// * @param problemBegin
-// * @param problemEnd
-// * @param sentenceType
-// * @return
-// */
-// public FSIterator<Annotation> createFilteredIteratorByBeginEndAndType(
-// int problemBegin, int problemEnd, Type sentenceType)
-// {
-// ConstraintFactory cf = jcas.getConstraintFactory();
-// TypeSystem ts = jcas.getTypeSystem();
-// Type annotationType = ts.getType(Annotation.class.getName());
-// Feature sentenceBeginFeature = annotationType.getFeatureByBaseName("begin");
-// FeaturePath sentenceBeginFeaturePath = jcas.createFeaturePath();
-// sentenceBeginFeaturePath.addFeature(sentenceBeginFeature);
-//
-// Feature sentenceEndFeature = annotationType.getFeatureByBaseName("end");
-// FeaturePath sentenceEndFeaturePath = jcas.createFeaturePath();
-// sentenceEndFeaturePath.addFeature(sentenceEndFeature);
-//
-// FSMatchConstraint beginAndEnd = constructContainedByConstraint(
-// problemBegin, problemEnd, cf, sentenceBeginFeaturePath,
-// sentenceEndFeaturePath);
-//
-//
-// FSTypeConstraint sentenceTypeConstraint = cf.createTypeConstraint();
-// sentenceTypeConstraint.add(sentenceType);
-//
-// FSMatchConstraint beginAndEndAndType = cf.and(beginAndEnd, sentenceTypeConstraint);
-//
-// FSIterator<Annotation> filteredIterator =
-// jcas.createFilteredIterator(jcas.getAnnotationIndex().iterator(), beginAndEndAndType);
-// return filteredIterator;
-// }
-//
-// /**
-// * @param problemBegin
-// * @param problemEnd
-// * @param cf
-// * @param sentenceBeginFeaturePath
-// * @param sentenceEndFeaturePath
-// * @return
-// */
-// public FSMatchConstraint constructContainedByConstraint(int problemBegin,
-// int problemEnd, ConstraintFactory cf,
-// FeaturePath sentenceBeginFeaturePath, FeaturePath sentenceEndFeaturePath)
-// {
-// FSIntConstraint sentenceBeginIntConstraint = cf.createIntConstraint();
-// sentenceBeginIntConstraint.leq(problemBegin);
-//
-// FSIntConstraint sentenceEndIntConstraint = cf.createIntConstraint();
-// sentenceEndIntConstraint.geq(problemEnd);
-//
-//
-// FSMatchConstraint begin = cf.embedConstraint(sentenceBeginFeaturePath, sentenceBeginIntConstraint);
-// FSMatchConstraint end = cf.embedConstraint(sentenceEndFeaturePath, sentenceEndIntConstraint);
-//
-// FSMatchConstraint beginAndEnd = cf.and(begin, end);
-// return beginAndEnd;
-// }
-
}