You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by pk...@apache.org on 2014/01/27 21:54:33 UTC
svn commit: r1561844 - in /uima/ruta/trunk/ruta-core/src:
main/java/org/apache/uima/ruta/resource/
test/java/org/apache/uima/ruta/action/
test/resources/org/apache/uima/ruta/action/
Author: pkluegl
Date: Mon Jan 27 20:54:33 2014
New Revision: 1561844
URL: http://svn.apache.org/r1561844
Log:
UIMA-3585
- extended test to cover bug
- incomplete matches that did not fail, should be continued -> multi token entities
- fixed duplicate annotations for anchoring ignored chars
Modified:
uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTextNode.java
uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTreeWordList.java
uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/TrieXMLEventHandler.java
uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/TrieTest.java
uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/TrieTest.ruta
uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/trie.mtwl
Modified: uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTextNode.java
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTextNode.java?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTextNode.java (original)
+++ uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTextNode.java Mon Jan 27 20:54:33 2014
@@ -48,10 +48,8 @@ public class MultiTextNode {
/** Documents/Types that contain the value specified by the super class. */
private Set<String> types;
- /**
- * Default-Kapazität, mit der die Kinder-Hashmap initialisiert wird. Also die Default-Anzahl der
- * Kinder, mit der wir rechnen.
- */
+ private boolean isWordEnd = false;
+
private static final int DEFAUL_INITIAL_CAPACITY = 5;
/** Default constructor uses a space as value. */
@@ -69,24 +67,7 @@ public class MultiTextNode {
* Indicates whether this is a word end.
*/
public MultiTextNode(char value, boolean isWordEnd) {
- this(value, isWordEnd, new String[] {});
- }
-
- /**
- * Constructs a MultiTextNode with a character and the information, whether this is a word end or
- * not. Documents are the documents, where the word, represented by the path from the root to this
- * node, occurs.
- *
- * @param value
- * The Character represented by the node.
- * @param isWordEnd
- * Indicates whether this is a word end.
- * @param documents
- * The documents, where the word, represented by the path from the root to this node,
- * occurs.
- */
- public MultiTextNode(char value, boolean isWordEnd, String... documents) {
- this(value, isWordEnd, DEFAUL_INITIAL_CAPACITY, documents);
+ this(value, isWordEnd, DEFAUL_INITIAL_CAPACITY);
}
/**
@@ -100,32 +81,11 @@ public class MultiTextNode {
* Indicates whether this is a word end.
* @param capacity
* The initial capacity of the map of child nodes.
- * @param documents
- * The documents, where the word, represented by the path from the root to this node,
- * occurs.
*/
- private MultiTextNode(char value, boolean isWordEnd, int capacity, String... documents) {
-
+ private MultiTextNode(char value, boolean isWordEnd, int capacity) {
this.value = value;
this.children = new HashMap<Character, MultiTextNode>(capacity);
setWordEnd(isWordEnd);
-
- for (String s : documents) {
- addType(s);
- }
- }
-
- /**
- * Adds an edge c to the map of children, which points to a new MultiTextNode with value c.
- *
- * @param c
- * The value of the new node.
- * @param isWordEnd
- * The isWordEnd-property of the new node.
- * @return The previous value associated with c, if exists, null otherwise.
- */
- public MultiTextNode addChild(char c, boolean isWordEnd) {
- return addChild(new MultiTextNode(c, false));
}
/**
@@ -136,15 +96,13 @@ public class MultiTextNode {
* @return The previous value associated with the value of n, if exists, null otherwise.
*/
public MultiTextNode addChild(MultiTextNode n) {
-
- // Lazy Initialization.
if (children == null) {
children = new HashMap<Character, MultiTextNode>();
}
-
return children.put(n.getValue(), n);
}
+
/**
* Returns the child, when you follow the c-edge, if exists. Returns null otherwise.
*
@@ -176,12 +134,12 @@ public class MultiTextNode {
}
/**
- * Returns a value which speficies, whether the word is finished or not.
+ * Returns a value which specifies, whether the word is finished or not.
*
- * @return A value which speficies, whether the word is finished or not.
+ * @return A value which specifies, whether the word is finished or not.
*/
public boolean isWordEnd() {
- return types != null && !types.isEmpty();
+ return isWordEnd;
}
/**
@@ -207,8 +165,6 @@ public class MultiTextNode {
* @return True, if the sources did not already contain the string document, false otherwise.
*/
public boolean addType(String type) {
-
- // Lazy Initialization.
if (types == null) {
types = new HashSet<String>();
}
@@ -236,21 +192,7 @@ public class MultiTextNode {
* The boolean, isWordEnd is set to.
*/
public void setWordEnd(boolean b) {
- setWordEnd(b, new String[] {});
- }
-
- /**
- * Sets isWordEnd to the specified boolean and adds a document to the types.
- *
- * @param b
- * The boolean, isWordEnd is set to.
- * @param document
- * The document which is added to the types.
- */
- public void setWordEnd(boolean b, String... documents) {
- for (String document : documents) {
- addType(document);
- }
+ isWordEnd = b;
}
/**
@@ -344,4 +286,5 @@ public class MultiTextNode {
return true;
}
+
}
Modified: uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTreeWordList.java
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTreeWordList.java?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTreeWordList.java (original)
+++ uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/MultiTreeWordList.java Mon Jan 27 20:54:33 2014
@@ -517,8 +517,8 @@ public class MultiTreeWordList implement
Map<String, Set<String>> resultMap = null;
if (!edit) {
- return recursiveContains2(root, string, 0, ignoreCase && string.length() > ignoreLength, true, ignoreToken.toCharArray(),
- ignoreLength);
+ return recursiveContains2(root, string, 0, ignoreCase && string.length() > ignoreLength,
+ true, ignoreToken.toCharArray(), ignoreLength);
} else {
if (string.length() >= ignoreLength && ignoreCase) {
resultMap = editDistance(string, (int) distance, true, ignoreToken, true);
@@ -538,6 +538,7 @@ public class MultiTreeWordList implement
return resultList;
}
+
/**
* Returns true, if the MultiTreeWordList contains the string text, false otherwise.
*
@@ -565,10 +566,10 @@ public class MultiTreeWordList implement
}
if (index == text.length()) {
- if(pointer.isWordEnd()) {
+ if (pointer.isWordEnd()) {
return new ArrayList<String>(pointer.getTypes());
}
- if(fragment) {
+ if (fragment) {
return Collections.emptyList();
}
}
@@ -592,25 +593,34 @@ public class MultiTreeWordList implement
// Lower Case Node.
MultiTextNode childNodeL = pointer.getChildNode(Character.toLowerCase(charAt));
-
+ if(childNodeL == null) {
+ childNodeL = skipWS(pointer, Character.toLowerCase(charAt));
+ }
+
// Upper Case Node.
MultiTextNode childNodeU = pointer.getChildNode(Character.toUpperCase(charAt));
-
+ if(childNodeU == null) {
+ childNodeU = skipWS(pointer, Character.toUpperCase(charAt));
+ }
+
if (charAtIgnored && childNodeL == null && childNodeU == null) {
// Character is ignored and does not appear.
return recursiveContains2(pointer, text, next, ignoreCase, fragment, ignoreChars,
maxIgnoreChars);
} else {
// Recursion.
- Collection<String> recursiveContainsL = recursiveContains2(childNodeL, text, next, ignoreCase, fragment, ignoreChars,
- maxIgnoreChars);
- Collection<String> recursiveContainsU = recursiveContains2(childNodeU, text, next, ignoreCase, fragment, ignoreChars,
- maxIgnoreChars);
+ Collection<String> recursiveContainsL = recursiveContains2(childNodeL, text, next,
+ ignoreCase, fragment, ignoreChars, maxIgnoreChars);
+ Collection<String> recursiveContainsU = recursiveContains2(childNodeU, text, next,
+ ignoreCase, fragment, ignoreChars, maxIgnoreChars);
+ if(recursiveContainsL == null && recursiveContainsU == null) {
+ return null;
+ }
List<String> result = new LinkedList<String>();
- if(recursiveContainsL != null) {
+ if (recursiveContainsL != null) {
result.addAll(recursiveContainsL);
}
- if(recursiveContainsU!= null) {
+ if (recursiveContainsU != null) {
result.addAll(recursiveContainsU);
}
return result;
@@ -631,6 +641,19 @@ public class MultiTreeWordList implement
}
}
}
+
+ private MultiTextNode skipWS(MultiTextNode pointer, char charAt) {
+ MultiTextNode childNode = pointer.getChildNode(' ');
+ if (childNode != null) {
+ MultiTextNode node = childNode.getChildNode(charAt);
+ if (node == null) {
+ return skipWS(childNode, charAt);
+ } else {
+ return node;
+ }
+ }
+ return null;
+ }
/**
* Returns true, if the MultiTreeWordList contains the string text, false otherwise.
@@ -729,47 +752,58 @@ public class MultiTreeWordList implement
String text = anchorBasic.getCoveredText();
StringBuilder candidate = new StringBuilder(text);
String lastCandidate = candidate.toString();
- List<AnnotationFS> interResults = new ArrayList<AnnotationFS>();
- while (streamPointer.isValid()) {
+ if (text.length() != 1 || !ignoreToken.contains(text)) {
- List<String> types = containsFragment(candidate.toString(), ignoreCase, ignoreLength, edit,
- distance, ignoreToken);
+ List<AnnotationFS> interResults = new ArrayList<AnnotationFS>();
- if (types != null && !types.isEmpty()) {
- streamPointer.moveToNext();
- if (streamPointer.isValid()) {
- RutaBasic next = (RutaBasic) streamPointer.get();
- // List<String> contains = contains(candidate,
- // ignoreCase,
- // ignoreLength, edit, distance, ignoreToken);
-
- tryToCreateAnnotation(types, stream, results, basicsToAdd, candidate.toString(), interResults,
- ignoreCase, ignoreLength, edit, distance, ignoreToken, typeMap);
- //
- lastCandidate = candidate.toString();
- candidate.append(next.getCoveredText());
- basicsToAdd.add(next);
+ while (streamPointer.isValid()) {
- } else {
- // !streamPointer.isValid();
- tryToCreateAnnotation(types, stream, results, basicsToAdd, lastCandidate, interResults,
- ignoreCase, ignoreLength, edit, distance, ignoreToken, typeMap);
+ boolean skip = false;
+ String currentBasicText = basicsToAdd.get(basicsToAdd.size()-1).getCoveredText();
+ if (currentBasicText.length() == 1 && ignoreToken.contains(currentBasicText)) {
+ skip = true;
}
- } else {
+ List<String> types = null;
+ if (!skip) {
+ types = containsFragment(candidate.toString(), ignoreCase, ignoreLength, edit,
+ distance, ignoreToken);
+ }
+ if (skip || types != null) {
+ streamPointer.moveToNext();
+ if (streamPointer.isValid()) {
+ RutaBasic next = (RutaBasic) streamPointer.get();
+ // List<String> contains = contains(candidate,
+ // ignoreCase,
+ // ignoreLength, edit, distance, ignoreToken);
+ if (!skip) {
+ tryToCreateAnnotation(types, stream, results, basicsToAdd, candidate.toString(),
+ interResults, ignoreCase, ignoreLength, edit, distance, ignoreToken,
+ typeMap);
+ }
+ lastCandidate = candidate.toString();
+ candidate.append(next.getCoveredText());
+ basicsToAdd.add(next);
+
+ } else {
+ // !streamPointer.isValid();
+ tryToCreateAnnotation(types, stream, results, basicsToAdd, lastCandidate,
+ interResults, ignoreCase, ignoreLength, edit, distance, ignoreToken, typeMap);
+ }
+ } else {
- // containsFragment.isEmpty();
- // basicsToAdd.remove(basicsToAdd.size() - 1);
- // tryToCreateAnnotation(stream, results, basicsToAdd,
- // lastCandidate, interResults, ignoreCase,
- // ignoreLength, edit, distance, ignoreToken, typeMap);
+ // containsFragment.isEmpty();
+ // basicsToAdd.remove(basicsToAdd.size() - 1);
+ // tryToCreateAnnotation(stream, results, basicsToAdd,
+ // lastCandidate, interResults, ignoreCase,
+ // ignoreLength, edit, distance, ignoreToken, typeMap);
- // breaks inner while()-loop.
- break;
- }
+ // breaks inner while()-loop.
+ break;
+ }
+ }
}
-
stream.moveToNext();
}
@@ -782,10 +816,10 @@ public class MultiTreeWordList implement
return new ArrayList<AnnotationFS>();
}
- private void tryToCreateAnnotation(List<String> types, RutaStream stream, Collection<AnnotationFS> results,
- List<RutaBasic> basicsToAdd, String lastCandidate, List<AnnotationFS> interResult,
- boolean ignoreCase, int ignoreLength, boolean edit, double distance, String ignoreToken,
- Map<String, Type> map) {
+ private void tryToCreateAnnotation(List<String> types, RutaStream stream,
+ Collection<AnnotationFS> results, List<RutaBasic> basicsToAdd, String lastCandidate,
+ List<AnnotationFS> interResult, boolean ignoreCase, int ignoreLength, boolean edit,
+ double distance, String ignoreToken, Map<String, Type> map) {
if (basicsToAdd.size() >= 1 && types != null) {
for (String each : types) {
Type type = map.get(each);
Modified: uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/TrieXMLEventHandler.java
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/TrieXMLEventHandler.java?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/TrieXMLEventHandler.java (original)
+++ uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/resource/TrieXMLEventHandler.java Mon Jan 27 20:54:33 2014
@@ -89,7 +89,6 @@ public class TrieXMLEventHandler extends
@Override
public void characters(char ch[], int start, int length) {
- // if (listeningState == true) {
if (stack.isEmpty()) {
return;
}
@@ -100,6 +99,7 @@ public class TrieXMLEventHandler extends
type.append(String.valueOf(ch[i]));
}
peek.addType(type.toString());
+ peek.setWordEnd(true);
} else if (inContent) {
if (ch.length > 0) {
peek.setValue(ch[0]);
@@ -107,7 +107,5 @@ public class TrieXMLEventHandler extends
peek.setValue(' ');
}
}
- // }
-
}
}
Modified: uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/TrieTest.java
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/TrieTest.java?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/TrieTest.java (original)
+++ uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/TrieTest.java Mon Jan 27 20:54:33 2014
@@ -64,7 +64,23 @@ public class TrieTest {
assertEquals("Kluegl", iterator.next().getCoveredText());
assertEquals("Schor", iterator.next().getCoveredText());
assertEquals("Kottmann", iterator.next().getCoveredText());
+
+ t = RutaTestUtils.getTestType(cas, 3);
+ ai = cas.getAnnotationIndex(t);
+ assertEquals(3, ai.size());
+ iterator = ai.iterator();
+ assertEquals("Peter Kluegl", iterator.next().getCoveredText());
+ assertEquals("Marshall Schor", iterator.next().getCoveredText());
+ assertEquals("Joern Kottmann", iterator.next().getCoveredText());
+ t = RutaTestUtils.getTestType(cas, 4);
+ ai = cas.getAnnotationIndex(t);
+ assertEquals(3, ai.size());
+ iterator = ai.iterator();
+ assertEquals("Peter Kluegl: Ruta", iterator.next().getCoveredText());
+ assertEquals("Marshall Schor: UIMA", iterator.next().getCoveredText());
+ assertEquals("Joern Kottmann: CAS Editor", iterator.next().getCoveredText());
+
cas.release();
}
Modified: uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/TrieTest.ruta
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/TrieTest.ruta?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/TrieTest.ruta (original)
+++ uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/TrieTest.ruta Mon Jan 27 20:54:33 2014
@@ -4,5 +4,6 @@ WORDLIST list1 = 'trie.mtwl';
DECLARE T1, T2, T3, T4, T5;
-Document{->TRIE("FirstNames.txt" = T1, "LastNames.txt" = T2,
- list1, true, 4, false, 0, ".,-/")};
+Document{->TRIE("FirstNames.txt" = T1, "LastNames.txt" = T2,
+ "CompleteNames.txt" = T3, "NamesWithSystems.txt" = T4,
+ list1, true, 4, false, 0, ":")};
Modified: uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/trie.mtwl
URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/trie.mtwl?rev=1561844&r1=1561843&r2=1561844&view=diff
==============================================================================
--- uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/trie.mtwl (original)
+++ uima/ruta/trunk/ruta-core/src/test/resources/org/apache/uima/ruta/action/trie.mtwl Mon Jan 27 20:54:33 2014
@@ -6,12 +6,35 @@
<n><c><![CDATA[h]]></c>
<n><c><![CDATA[a]]></c>
<n><c><![CDATA[l]]></c>
-<n><c><![CDATA[l]]></c><t>FirstNames.txt</t></n></n></n></n></n></n></n></n>
+<n><c><![CDATA[l]]></c><t>FirstNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[S]]></c>
+<n><c><![CDATA[c]]></c>
+<n><c><![CDATA[h]]></c>
+<n><c><![CDATA[o]]></c>
+<n><c><![CDATA[r]]></c><t>CompleteNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[U]]></c>
+<n><c><![CDATA[I]]></c>
+<n><c><![CDATA[M]]></c>
+<n><c><![CDATA[A]]></c><t>NamesWithSystems.txt</t></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n>
<n><c><![CDATA[P]]></c>
<n><c><![CDATA[e]]></c>
<n><c><![CDATA[t]]></c>
<n><c><![CDATA[e]]></c>
-<n><c><![CDATA[r]]></c><t>FirstNames.txt</t></n></n></n></n></n>
+<n><c><![CDATA[r]]></c><t>FirstNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[K]]></c>
+<n><c><![CDATA[l]]></c>
+<n><c><![CDATA[u]]></c>
+<n><c><![CDATA[e]]></c>
+<n><c><![CDATA[g]]></c>
+<n><c><![CDATA[l]]></c><t>CompleteNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[R]]></c>
+<n><c><![CDATA[u]]></c>
+<n><c><![CDATA[t]]></c>
+<n><c><![CDATA[a]]></c><t>NamesWithSystems.txt</t></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n>
<n><c><![CDATA[S]]></c>
<n><c><![CDATA[c]]></c>
<n><c><![CDATA[h]]></c>
@@ -21,7 +44,27 @@
<n><c><![CDATA[o]]></c>
<n><c><![CDATA[e]]></c>
<n><c><![CDATA[r]]></c>
-<n><c><![CDATA[n]]></c><t>FirstNames.txt</t></n></n></n></n></n>
+<n><c><![CDATA[n]]></c><t>FirstNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[K]]></c>
+<n><c><![CDATA[o]]></c>
+<n><c><![CDATA[t]]></c>
+<n><c><![CDATA[t]]></c>
+<n><c><![CDATA[m]]></c>
+<n><c><![CDATA[a]]></c>
+<n><c><![CDATA[n]]></c>
+<n><c><![CDATA[n]]></c><t>CompleteNames.txt</t>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[C]]></c>
+<n><c><![CDATA[A]]></c>
+<n><c><![CDATA[S]]></c>
+<n><c><![CDATA[ ]]></c>
+<n><c><![CDATA[E]]></c>
+<n><c><![CDATA[d]]></c>
+<n><c><![CDATA[i]]></c>
+<n><c><![CDATA[t]]></c>
+<n><c><![CDATA[o]]></c>
+<n><c><![CDATA[r]]></c><t>NamesWithSystems.txt</t></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n></n>
<n><c><![CDATA[K]]></c>
<n><c><![CDATA[o]]></c>
<n><c><![CDATA[t]]></c>