You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ctakes.apache.org by vj...@apache.org on 2014/01/06 16:03:39 UTC
svn commit: r1555844 -
/ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
Author: vjapache
Date: Mon Jan 6 15:03:38 2014
New Revision: 1555844
URL: http://svn.apache.org/r1555844
Log:
remove newlines within sentences from the token list
Modified:
ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
Modified: ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
URL: http://svn.apache.org/viewvc/ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java?rev=1555844&r1=1555843&r2=1555844&view=diff
==============================================================================
--- ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java (original)
+++ ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java Mon Jan 6 15:03:38 2014
@@ -30,6 +30,7 @@ import org.apache.ctakes.core.resource.F
import org.apache.ctakes.core.util.ListFactory;
import org.apache.ctakes.typesystem.type.syntax.BaseToken;
import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
+import org.apache.ctakes.typesystem.type.syntax.NewlineToken;
import org.apache.ctakes.typesystem.type.textsem.Predicate;
import org.apache.ctakes.typesystem.type.textsem.SemanticArgument;
import org.apache.ctakes.typesystem.type.textsem.SemanticRoleRelation;
@@ -169,30 +170,36 @@ final String language = AbstractReader.L
int[] headIDs = new int[tokens.size()];
String[] deprels = new String[tokens.size()];
- // Initialize Token / Sentence info for the ClearNLP Semantic Role Labeler
+ // Initialize Token / Sentence info for the ClearNLP Semantic Role Labeler
+ // we are filtering out newline tokens
+ // use idIter as the non-newline token index counter
+ int idIter = 0;
for (int i = 0; i < tokens.size(); i++) {
- BaseToken token = tokens.get(i);
-
- // Determine HeadId
- List<ConllDependencyNode> casDepNodes = JCasUtil.selectCovered(jCas, ConllDependencyNode.class, token);
-
- ConllDependencyNode casDepNode = casDepNodes.get(0);
- if(casDepNode.getId() == 0) casDepNode = casDepNodes.get(1);
-
- deprels[i] = casDepNode.getDeprel();
- ConllDependencyNode head = casDepNode.getHead();
-
- // If there is no head, this is the head node, set node to 0
- headIDs[i] = (head == null) ? 0 : depNodeToID.get(head);
-
- // Populate Dependency Node / Tree information
- int id = i + 1;
- String form = casDepNode.getForm();
- String pos = casDepNode.getPostag();
- String lemma = casDepNode.getLemma();
-
- DEPNode node = new DEPNode(id, form, lemma, pos, new DEPFeat());
- tree.add(node);
+ BaseToken token = tokens.get(i);
+ // ignore newline tokens within a sentence - newline = whitespace = non-token
+ if(!(token instanceof NewlineToken)) {
+ // Determine HeadId
+ List<ConllDependencyNode> casDepNodes = JCasUtil.selectCovered(jCas, ConllDependencyNode.class, token);
+
+ ConllDependencyNode casDepNode = casDepNodes.get(0);
+ if(casDepNode.getId() == 0) casDepNode = casDepNodes.get(1);
+
+ deprels[i] = casDepNode.getDeprel();
+ ConllDependencyNode head = casDepNode.getHead();
+
+ // If there is no head, this is the head node, set node to 0
+ headIDs[i] = (head == null) ? 0 : depNodeToID.get(head);
+
+ // Populate Dependency Node / Tree information
+ int id = idIter + 1;
+ String form = casDepNode.getForm();
+ String pos = casDepNode.getPostag();
+ String lemma = casDepNode.getLemma();
+
+ DEPNode node = new DEPNode(id, form, lemma, pos, new DEPFeat());
+ tree.add(node);
+ idIter++;
+ }
}
for (int i=1; i<tree.size(); i++)