You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/01/26 12:30:07 UTC
svn commit: r1561481 - in /opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
main/java/opennlp/tools/util/Span.java
test/java/opennlp/tools/util/SpanTest.java
Author: joern
Date: Sun Jan 26 11:30:06 2014
New Revision: 1561481
URL: http://svn.apache.org/r1561481
Log:
OPENNLP-602 Fixed handling of white space only sentences.
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/Span.java
opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/SpanTest.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java?rev=1561481&r1=1561480&r2=1561481&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java Sun Jan 26 11:30:06 2014
@@ -40,7 +40,7 @@ import opennlp.tools.util.model.ModelUti
/**
* A sentence detector for splitting up raw text into sentences.
* <p>
- * A maximum entropy model is used to evaluate the characters ".", "!", and "?" in a
+ * A maximum entropy model is used to evaluate end-of-sentence characters in a
* string to determine if they signify the end of a sentence.
*/
public class SentenceDetectorME implements SentenceDetector {
@@ -54,9 +54,6 @@ public class SentenceDetectorME implemen
* Constant indicates no sentence split.
*/
public static final String NO_SPLIT ="n";
-
- // Note: That should be inlined when doing a re-factoring!
- private static final Double ONE = new Double(1);
/**
* The maximum entropy model to use to evaluate contexts.
@@ -223,33 +220,41 @@ public class SentenceDetectorME implemen
return new Span[0];
}
- // Now convert the sent indexes to spans
+ // Convert the sentence end indexes to spans
+
boolean leftover = starts[starts.length - 1] != s.length();
- Span[] spans = new Span[leftover? starts.length + 1 : starts.length];
- for (int si=0;si<starts.length;si++) {
- int start,end;
+ List<Span> spans = new ArrayList<Span>(leftover? starts.length + 1 : starts.length);
+
+ for (int si=0; si < starts.length; si++) {
+ int start;
+
if (si==0) {
start = 0;
-
- while (si < starts.length && StringUtil.isWhitespace(s.charAt(start)))
- start++;
}
else {
start = starts[si-1];
}
- end = starts[si];
- while (end > 0 && StringUtil.isWhitespace(s.charAt(end-1))) {
- end--;
+
+ // A span might contain only white spaces, in this case the length of
+ // the span will be zero after trimming and should be ignored.
+ Span span = new Span(start, starts[si]).trim(s);
+ if (span.length() > 0) {
+ spans.add(span);
+ }
+ else {
+ sentProbs.remove(si);
}
- spans[si]=new Span(start,end);
}
if (leftover) {
- spans[spans.length-1] = new Span(starts[starts.length-1],s.length());
- sentProbs.add(ONE);
+ Span span = new Span(starts[starts.length-1],s.length()).trim(s);
+ if (span.length() > 0) {
+ spans.add(span);
+ sentProbs.add(1d);
+ }
}
- return spans;
+ return spans.toArray(new Span[spans.size()]);
}
/**
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/Span.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/Span.java?rev=1561481&r1=1561480&r2=1561481&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/Span.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/Span.java Sun Jan 26 11:30:06 2014
@@ -18,6 +18,8 @@
package opennlp.tools.util;
+import opennlp.tools.sentdetect.EndOfSentenceScanner;
+
/**
* Class for storing start and end integer offsets.
**/
@@ -222,6 +224,9 @@ public class Span implements Comparable<
if (newStartOffset == getStart() && newEndOffset == getEnd()) {
return this;
}
+ else if (newStartOffset > newEndOffset) {
+ return new Span(getStart(), getStart(), getType());
+ }
else {
return new Span(newStartOffset, newEndOffset, getType());
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/SpanTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/SpanTest.java?rev=1561481&r1=1561480&r2=1561481&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/SpanTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/SpanTest.java Sun Jan 26 11:30:06 2014
@@ -318,4 +318,11 @@ public class SpanTest {
Span span1 = new Span(0, string1.length());
assertEquals("12 34", span1.trim(string1).getCoveredText(string1));
}
+
+ @Test
+ public void testTrimWhitespaceSpan() {
+ String string1 = " ";
+ Span span1 = new Span(0, string1.length());
+ assertEquals("", span1.trim(string1).getCoveredText(string1));
+ }
}