You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/05/02 14:34:33 UTC
svn commit: r1591889 [11/14] - in /opennlp/trunk: opennlp-tools/lang/ml/
opennlp-tools/src/main/java/opennlp/tools/chunker/
opennlp-tools/src/main/java/opennlp/tools/cmdline/
opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/
opennlp-tools/src/...
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -33,130 +33,130 @@ public class ConllXPOSSampleStreamTest {
@Test
public void testParsingSample() throws IOException {
-
- InputStreamFactory in = new ResourceAsStreamFactory(ConllXPOSSampleStreamTest.class,
+
+ InputStreamFactory in = new ResourceAsStreamFactory(ConllXPOSSampleStreamTest.class,
"/opennlp/tools/formats/conllx.sample");
-
+
ObjectStream<POSSample> sampleStream = new ConllXPOSSampleStream(in,Charset.forName("UTF-8"));
-
+
POSSample a = sampleStream.read();
-
+
String aSentence[] = a.getSentence();
String aTags[] = a.getTags();
-
+
assertEquals(22, aSentence.length);
assertEquals(22, aTags.length);
-
+
assertEquals("To", aSentence[0]);
assertEquals("AC", aTags[0]);
-
+
assertEquals("kendte", aSentence[1]);
assertEquals("AN", aTags[1]);
-
+
assertEquals("russiske", aSentence[2]);
assertEquals("AN", aTags[2]);
-
+
assertEquals("historikere", aSentence[3]);
assertEquals("NC", aTags[3]);
-
+
assertEquals("Andronik", aSentence[4]);
assertEquals("NP", aTags[4]);
-
+
assertEquals("Andronik", aSentence[5]);
assertEquals("NP", aTags[5]);
-
+
assertEquals("og", aSentence[6]);
assertEquals("CC", aTags[6]);
-
+
assertEquals("Igor", aSentence[7]);
assertEquals("NP", aTags[7]);
-
+
assertEquals("Klamkin", aSentence[8]);
assertEquals("NP", aTags[8]);
-
+
assertEquals("tror", aSentence[9]);
assertEquals("VA", aTags[9]);
-
+
assertEquals("ikke", aSentence[10]);
assertEquals("RG", aTags[10]);
assertEquals(",", aSentence[11]);
assertEquals("XP", aTags[11]);
-
+
assertEquals("at", aSentence[12]);
assertEquals("CS", aTags[12]);
-
+
assertEquals("Rusland", aSentence[13]);
assertEquals("NP", aTags[13]);
-
+
assertEquals("kan", aSentence[14]);
assertEquals("VA", aTags[14]);
-
+
assertEquals("udvikles", aSentence[15]);
assertEquals("VA", aTags[15]);
-
+
assertEquals("uden", aSentence[16]);
assertEquals("SP", aTags[16]);
-
+
assertEquals("en", aSentence[17]);
assertEquals("PI", aTags[17]);
-
+
assertEquals("\"", aSentence[18]);
assertEquals("XP", aTags[18]);
-
+
assertEquals("jernnæve", aSentence[19]);
assertEquals("NC", aTags[19]);
-
+
assertEquals("\"", aSentence[20]);
assertEquals("XP", aTags[20]);
-
+
assertEquals(".", aSentence[21]);
assertEquals("XP", aTags[21]);
-
+
POSSample b = sampleStream.read();
-
+
String bSentence[] = b.getSentence();
String bTags[] = b.getTags();
-
+
assertEquals(12, bSentence.length);
assertEquals(12, bTags.length);
-
+
assertEquals("De", bSentence[0]);
assertEquals("PP", bTags[0]);
-
+
assertEquals("hævder", bSentence[1]);
assertEquals("VA", bTags[1]);
-
+
assertEquals(",", bSentence[2]);
assertEquals("XP", bTags[2]);
-
+
assertEquals("at", bSentence[3]);
assertEquals("CS", bTags[3]);
-
+
assertEquals("Ruslands", bSentence[4]);
assertEquals("NP", bTags[4]);
-
+
assertEquals("vej", bSentence[5]);
assertEquals("NC", bTags[5]);
-
+
assertEquals("til", bSentence[6]);
assertEquals("SP", bTags[6]);
-
+
assertEquals("demokrati", bSentence[7]);
assertEquals("NC", bTags[7]);
-
+
assertEquals("går", bSentence[8]);
assertEquals("VA", bTags[8]);
-
+
assertEquals("gennem", bSentence[9]);
assertEquals("SP", bTags[9]);
-
+
assertEquals("diktatur", bSentence[10]);
assertEquals("NC", bTags[10]);
-
+
assertEquals(".", bSentence[11]);
assertEquals("XP", bTags[11]);
-
+
assertNull(sampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -32,50 +32,50 @@ import opennlp.tools.util.Span;
import org.junit.Test;
/**
- *
+ *
* Note:
* Sample training data must be UTF-8 encoded and uncompressed!
*/
public class EvalitaNameSampleStreamTest {
-
+
private static ObjectStream<NameSample> openData(LANGUAGE lang, String name) throws IOException {
- InputStreamFactory in = new ResourceAsStreamFactory(EvalitaNameSampleStreamTest.class,
+ InputStreamFactory in = new ResourceAsStreamFactory(EvalitaNameSampleStreamTest.class,
"/opennlp/tools/formats/" + name);
-
+
return new EvalitaNameSampleStream(lang, in, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES);
}
-
+
@Test
public void testParsingItalianSample() throws IOException {
-
+
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.IT, "evalita-ner-it.sample");
-
+
NameSample personName = sampleStream.read();
-
+
assertNotNull(personName);
-
+
assertEquals(11, personName.getSentence().length);
assertEquals(1, personName.getNames().length);
assertEquals(true, personName.isClearAdaptiveDataSet());
-
+
Span nameSpan = personName.getNames()[0];
assertEquals(8, nameSpan.getStart());
assertEquals(10, nameSpan.getEnd());
assertEquals(true, personName.isClearAdaptiveDataSet());
-
+
assertEquals(0, sampleStream.read().getNames().length);
-
+
assertNull(sampleStream.read());
}
-
+
@Test
public void testReset() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.IT, "evalita-ner-it.sample");
-
+
NameSample sample = sampleStream.read();
-
+
sampleStream.reset();
-
+
assertEquals(sample, sampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/LeipzigDoccatSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/LeipzigDoccatSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/LeipzigDoccatSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/LeipzigDoccatSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -34,22 +34,22 @@ public class LeipzigDoccatSampleStreamTe
public void testParsingSample() throws IOException {
InputStream in = LeipzigDoccatSampleStreamTest.class.getResourceAsStream(
"/opennlp/tools/formats/leipzig-en.sample");
-
- ObjectStream<DocumentSample> sampleStream =
+
+ ObjectStream<DocumentSample> sampleStream =
new LeipzigDoccatSampleStream("en", 2, in);
-
+
DocumentSample doc1 = sampleStream.read();
assertEquals("en", doc1.getCategory());
-
+
DocumentSample doc2 = sampleStream.read();
assertEquals("en", doc2.getCategory());
-
+
DocumentSample doc3 = sampleStream.read();
assertEquals("en", doc3.getCategory());
DocumentSample doc4 = sampleStream.read();
assertEquals("en", doc4.getCategory());
-
+
assertNull(sampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/NameFinderCensus90NameStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/NameFinderCensus90NameStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/NameFinderCensus90NameStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/NameFinderCensus90NameStreamTest.java Fri May 2 12:34:23 2014
@@ -2,9 +2,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ResourceAsStreamFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ResourceAsStreamFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ResourceAsStreamFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ResourceAsStreamFactory.java Fri May 2 12:34:23 2014
@@ -2,9 +2,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,20 +26,20 @@ public class ResourceAsStreamFactory imp
private String name;
/**
- *
+ *
* @param clazz
* @param name
*/
public ResourceAsStreamFactory(Class<?> clazz, String name) {
-
+
if (clazz == null || name == null) {
throw new IllegalArgumentException("Null parameters are not allowed!");
}
-
+
this.clazz = clazz;
this.name = name;
}
-
+
@Override
public InputStream createInputStream() throws IOException {
return clazz.getResourceAsStream(name);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -58,7 +58,7 @@ public class ADChunkSampleStreamTest {
assertEquals("próximo", samples.get(0).getSentence()[3]);
assertEquals("adj", samples.get(0).getTags()[3]);
assertEquals("I-NP", samples.get(0).getPreds()[3]);
-
+
assertEquals("Casas", samples.get(3).getSentence()[0]);
assertEquals("n", samples.get(3).getTags()[0]);
assertEquals("B-NP", samples.get(3).getPreds()[0]);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -40,10 +40,10 @@ public class ADNameSampleStreamTest {
public void testSimpleCount() throws IOException {
assertEquals(ADParagraphStreamTest.NUM_SENTENCES, samples.size());
}
-
+
@Test
public void testCheckMergedContractions() throws IOException {
-
+
assertEquals("no", samples.get(0).getSentence()[1]);
assertEquals("no", samples.get(0).getSentence()[11]);
assertEquals("Com", samples.get(1).getSentence()[0]);
@@ -53,9 +53,9 @@ public class ADNameSampleStreamTest {
assertEquals("de", samples.get(2).getSentence()[5]);
assertEquals("da", samples.get(2).getSentence()[8]);
assertEquals("num", samples.get(3).getSentence()[26]);
-
+
}
-
+
@Test
public void testSize() throws IOException {
assertEquals(25, samples.get(0).getSentence().length);
@@ -63,7 +63,7 @@ public class ADNameSampleStreamTest {
assertEquals(59, samples.get(2).getSentence().length);
assertEquals(33, samples.get(3).getSentence().length);
}
-
+
@Test
public void testNames() throws IOException {
@@ -74,7 +74,7 @@ public class ADNameSampleStreamTest {
assertEquals(new Span(18, 19, "numeric"), samples.get(0).getNames()[4]);
assertEquals(new Span(20, 22, "place"), samples.get(0).getNames()[5]);
assertEquals(new Span(23, 24, "place"), samples.get(0).getNames()[6]);
-
+
assertEquals(new Span(22, 24, "person"), samples.get(2).getNames()[0]);// 22..24
assertEquals(new Span(25, 27, "person"), samples.get(2).getNames()[1]);// 25..27
assertEquals(new Span(28, 30, "person"), samples.get(2).getNames()[2]);// 28..30
@@ -86,24 +86,24 @@ public class ADNameSampleStreamTest {
assertEquals(new Span(47, 49, "person"), samples.get(2).getNames()[8]);// 47..49
assertEquals(new Span(50, 52, "person"), samples.get(2).getNames()[9]);// 50..52
assertEquals(new Span(53, 55, "person"), samples.get(2).getNames()[10]);// 53..55
-
+
assertEquals(new Span(0, 1, "place"), samples.get(3).getNames()[0]);// 0..1
assertEquals(new Span(6, 7, "event"), samples.get(3).getNames()[1]);// 6..7
assertEquals(new Span(15, 16, "organization"), samples.get(3).getNames()[2]);// 15..16
assertEquals(new Span(18, 19, "event"), samples.get(3).getNames()[3]);// 18..19
assertEquals(new Span(27, 28, "event"), samples.get(3).getNames()[4]);// 27..28
assertEquals(new Span(29, 30, "event"), samples.get(3).getNames()[5]);// 29..30
-
+
assertEquals(new Span(1, 6, "time"), samples.get(4).getNames()[0]);// 0..1
-
+
assertEquals(new Span(0, 3, "person"), samples.get(5).getNames()[0]);// 0..1
}
-
+
@Test
public void testSmallSentence() throws IOException {
assertEquals(2, samples.get(6).getSentence().length);
}
-
+
@Test
public void testMissingRightContraction() throws IOException {
assertEquals(new Span(0, 1, "person"), samples.get(7).getNames()[0]);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -38,22 +38,22 @@ public class ADPOSSampleStreamTest {
"UTF-8"), false, false);
POSSample sample = stream.read();
-
+
assertEquals(23, sample.getSentence().length);
-
+
assertEquals("Inicia", sample.getSentence()[0]);
assertEquals("v-fin", sample.getTags()[0]);
-
+
assertEquals("em", sample.getSentence()[1]);
assertEquals("prp", sample.getTags()[1]);
-
+
assertEquals("o", sample.getSentence()[2]);
assertEquals("art", sample.getTags()[2]);
-
+
assertEquals("Porto_Poesia", sample.getSentence()[9]);
assertEquals("prop", sample.getTags()[9]);
}
-
+
@Test
public void testExpandME() throws IOException {
// add one sentence with expandME = true
@@ -63,25 +63,25 @@ public class ADPOSSampleStreamTest {
"UTF-8"), true, false);
POSSample sample = stream.read();
-
+
assertEquals(27, sample.getSentence().length);
-
+
assertEquals("Inicia", sample.getSentence()[0]);
assertEquals("v-fin", sample.getTags()[0]);
-
+
assertEquals("em", sample.getSentence()[1]);
assertEquals("prp", sample.getTags()[1]);
-
+
assertEquals("o", sample.getSentence()[2]);
assertEquals("art", sample.getTags()[2]);
-
+
assertEquals("Porto", sample.getSentence()[9]);
assertEquals("B-prop", sample.getTags()[9]);
-
+
assertEquals("Poesia", sample.getSentence()[10]);
assertEquals("I-prop", sample.getTags()[10]);
}
-
+
@Test
public void testIncludeFeats() throws IOException {
// add one sentence with includeFeats = true
@@ -91,18 +91,18 @@ public class ADPOSSampleStreamTest {
"UTF-8"), false, true);
POSSample sample = stream.read();
-
+
assertEquals(23, sample.getSentence().length);
-
+
assertEquals("Inicia", sample.getSentence()[0]);
assertEquals("v-fin=PR=3S=IND=VFIN", sample.getTags()[0]);
-
+
assertEquals("em", sample.getSentence()[1]);
assertEquals("prp", sample.getTags()[1]);
-
+
assertEquals("o", sample.getSentence()[2]);
assertEquals("art=DET=M=S", sample.getTags()[2]);
-
+
assertEquals("Porto_Poesia", sample.getSentence()[9]);
assertEquals("prop=M=S", sample.getTags()[9]);
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java Fri May 2 12:34:23 2014
@@ -34,9 +34,9 @@ public class ADParagraphStreamTest {
@Test
public void testSimpleReading() throws IOException {
int count = 0;
-
+
ADSentenceStream stream = openData();
-
+
ADSentenceStream.Sentence paragraph = stream.read();
paragraph.getRoot();
while(paragraph != null) {
@@ -44,29 +44,29 @@ public class ADParagraphStreamTest {
paragraph = stream.read();
// paragraph.getRoot();
}
-
+
assertEquals(ADParagraphStreamTest.NUM_SENTENCES, count);
}
-
+
@Test
public void testLeadingWithContraction() throws IOException {
int count = 0;
-
+
ADSentenceStream stream = openData();
-
+
ADSentenceStream.Sentence paragraph = stream.read();
while(paragraph != null) {
-
+
count++;
paragraph = stream.read();
}
-
+
assertEquals(ADParagraphStreamTest.NUM_SENTENCES, count);
}
-
+
private static ADSentenceStream openData() throws IOException {
InputStreamFactory in = new ResourceAsStreamFactory(ADParagraphStreamTest.class, "/opennlp/tools/formats/ad.sample");
-
+
return new ADSentenceStream(new PlainTextByLineStream(in, "UTF-8"));
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -44,7 +44,7 @@ public class ADSentenceSampleStreamTest
@Test
public void testSentences() throws IOException {
-
+
assertNotNull(samples.get(0).getDocument());
assertEquals(3, samples.get(0).getSentences().length);
assertEquals(new Span(0, 119), samples.get(0).getSentences()[0]);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java Fri May 2 12:34:23 2014
@@ -26,57 +26,57 @@ import opennlp.tools.util.ObjectStream;
import org.junit.Test;
public class BratAnnotationStreamTest {
-
+
private ObjectStream<BratAnnotation> creatBratAnnotationStream(
AnnotationConfiguration conf, String file) {
-
+
InputStream in = BratAnnotationStreamTest.class.getResourceAsStream(
file);
-
+
return new BratAnnotationStream(conf, "testing", in);
}
-
-
+
+
static void addEntityTypes(Map<String, String> typeToClassMap) {
typeToClassMap.put("Person", AnnotationConfiguration.ENTITY_TYPE);
typeToClassMap.put("Location", AnnotationConfiguration.ENTITY_TYPE);
typeToClassMap.put("Organization", AnnotationConfiguration.ENTITY_TYPE);
typeToClassMap.put("Date", AnnotationConfiguration.ENTITY_TYPE);
}
-
+
@Test
public void testParsingEntities() throws Exception {
-
+
Map<String, String> typeToClassMap = new HashMap<String, String>();
addEntityTypes(typeToClassMap);
-
+
AnnotationConfiguration annConfig = new AnnotationConfiguration(typeToClassMap);
-
- ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
+
+ ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
"/opennlp/tools/formats/brat/voa-with-entities.ann");
-
+
// TODO: Test if we get the entities ... we expect!
-
+
BratAnnotation ann;
while ((ann = annStream.read()) != null) {
System.out.println(ann);
}
}
-
+
@Test
public void testParsingRelations() throws Exception {
-
+
Map<String, String> typeToClassMap = new HashMap<String, String>();
addEntityTypes(typeToClassMap);
typeToClassMap.put("Related", AnnotationConfiguration.RELATION_TYPE);
-
+
AnnotationConfiguration annConfig = new AnnotationConfiguration(typeToClassMap);
-
- ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
+
+ ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
"/opennlp/tools/formats/brat/voa-with-relations.ann");
-
+
// TODO: Test if we get the entities ... we expect!
-
+
BratAnnotation ann;
while ((ann = annStream.read()) != null) {
System.out.println(ann);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratDocumentTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratDocumentTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratDocumentTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratDocumentTest.java Fri May 2 12:34:23 2014
@@ -31,23 +31,23 @@ public class BratDocumentTest {
@Test
public void testDocumentWithEntitiesParsing() throws IOException {
-
+
Map<String, String> typeToClassMap = new HashMap<String, String>();
BratAnnotationStreamTest.addEntityTypes(typeToClassMap);
AnnotationConfiguration config = new AnnotationConfiguration(typeToClassMap);
-
+
InputStream txtIn = BratDocumentTest.class.getResourceAsStream(
"/opennlp/tools/formats/brat/voa-with-entities.txt");
-
+
InputStream annIn = BratDocumentTest.class.getResourceAsStream(
"/opennlp/tools/formats/brat/voa-with-entities.ann");
-
+
BratDocument doc = BratDocument.parseDocument(config, "voa-with-entities", txtIn, annIn);
assertEquals("voa-with-entities", doc.getId());
assertTrue(doc.getText().startsWith(" U . S . President "));
assertTrue(doc.getText().endsWith("multinational process . \n"));
-
+
assertEquals(18, doc.getAnnotations().size());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -79,18 +79,18 @@ public class ConstitParseSampleStreamTes
"Allemagne",
"."
};
-
+
/**
* Reads sample1.xml into a byte array.
- *
+ *
* @return byte array containing sample1.xml.
*/
static byte[] getSample1() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
-
+
InputStream sampleIn =
ConstitParseSampleStreamTest.class.getResourceAsStream("sample1.xml");
-
+
byte buffer[] = new byte[1024];
int length;
try {
@@ -100,34 +100,34 @@ public class ConstitParseSampleStreamTes
} finally {
sampleIn.close();
}
-
+
return out.toByteArray();
}
@Test
public void testThereIsExactlyOneSent() throws IOException {
- ObjectStream<Parse> samples =
+ ObjectStream<Parse> samples =
new ConstitParseSampleStream(ObjectStreamUtils.createObjectStream(getSample1()));
-
+
Assert.assertNotNull(samples.read());
Assert.assertNull(samples.read());
Assert.assertNull(samples.read());
}
-
+
@Test
public void testTokensAreCorrect() throws IOException {
-
- ObjectStream<Parse> samples =
+
+ ObjectStream<Parse> samples =
new ConstitParseSampleStream(ObjectStreamUtils.createObjectStream(getSample1()));
-
+
Parse p = samples.read();
-
+
Parse[] tagNodes = p.getTagNodes();
String[] tokens = new String[tagNodes.length];
for (int ti=0;ti<tagNodes.length;ti++){
tokens[ti] = tagNodes[ti].getCoveredText();
}
-
+
Assert.assertArrayEquals(sample1Tokens, tokens);
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/DocumentSplitterStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/DocumentSplitterStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/DocumentSplitterStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/DocumentSplitterStreamTest.java Fri May 2 12:34:23 2014
@@ -29,7 +29,7 @@ public class DocumentSplitterStreamTest
@Test
public void testSplitTwoDocuments() throws IOException {
-
+
StringBuilder docsString = new StringBuilder();
for (int i = 0; i < 2; i++) {
@@ -37,18 +37,18 @@ public class DocumentSplitterStreamTest
docsString.append("test document #"+ i + "\n");
docsString.append("</DOC>\n");
}
-
+
ObjectStream<String> docs = new DocumentSplitterStream(
ObjectStreamUtils.createObjectStream(docsString.toString()));
-
+
String doc1 = docs.read();
Assert.assertEquals(docsString.length() / 2, doc1.length() + 1);
Assert.assertTrue(doc1.contains("#0"));
-
+
String doc2 = docs.read();
Assert.assertEquals(docsString.length() / 2, doc2.length() + 1);
Assert.assertTrue(doc2.contains("#1"));
-
+
Assert.assertNull(docs.read());
Assert.assertNull(docs.read());
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/SgmlParserTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/SgmlParserTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/SgmlParserTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/muc/SgmlParserTest.java Fri May 2 12:34:23 2014
@@ -29,7 +29,7 @@ public class SgmlParserTest {
public void testParse1() throws IOException {
Reader in = new InputStreamReader(SgmlParserTest.class.getResourceAsStream("parsertest1.sgml"), "UTF-8");
-
+
try {
SgmlParser parser = new SgmlParser();
parser.parse(in, new SgmlParser.ContentHandler() {
@@ -38,7 +38,7 @@ public class SgmlParserTest {
finally {
in.close();
}
-
+
}
-
+
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/PrepAttachDataUtil.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/PrepAttachDataUtil.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/PrepAttachDataUtil.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/PrepAttachDataUtil.java Fri May 2 12:34:23 2014
@@ -40,7 +40,7 @@ public class PrepAttachDataUtil {
InputStream in = PerceptronPrepAttachTest.class.getResourceAsStream("/data/ppa/" +
filename);
-
+
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
String line;
@@ -55,15 +55,15 @@ public class PrepAttachDataUtil {
finally {
in.close();
}
-
+
return events;
}
-
+
public static ObjectStream<Event> createTrainingStream() throws IOException {
List<Event> trainingEvents = readPpaFile("training");
return ObjectStreamUtils.createObjectStream(trainingEvents);
}
-
+
public static void testModel(MaxentModel model, double expecedAccuracy) throws IOException {
List<Event> devEvents = readPpaFile("devset");
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/TrainerFactoryTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/TrainerFactoryTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/TrainerFactoryTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/TrainerFactoryTest.java Fri May 2 12:34:23 2014
@@ -27,7 +27,7 @@ import org.junit.Before;
import org.junit.Test;
public class TrainerFactoryTest {
-
+
private TrainingParameters mlParams;
@Before
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/MaxentPrepAttachTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/MaxentPrepAttachTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/MaxentPrepAttachTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/MaxentPrepAttachTest.java Fri May 2 12:34:23 2014
@@ -38,46 +38,46 @@ public class MaxentPrepAttachTest {
@Test
public void testMaxentOnPrepAttachData() throws IOException {
- AbstractModel model =
- new GISTrainer(true).trainModel(100,
+ AbstractModel model =
+ new GISTrainer(true).trainModel(100,
new TwoPassDataIndexer(createTrainingStream(), 1), 1);
testModel(model, 0.7997028967566229);
}
-
+
@Test
public void testMaxentOnPrepAttachData2Threads() throws IOException {
- AbstractModel model =
+ AbstractModel model =
new GISTrainer(true).trainModel(100,
new TwoPassDataIndexer(createTrainingStream(), 1),
new UniformPrior(), 1, 2);
-
+
testModel(model, 0.7997028967566229);
}
-
+
@Test
public void testMaxentOnPrepAttachDataWithParams() throws IOException {
-
+
Map<String, String> trainParams = new HashMap<String, String>();
trainParams.put(AbstractTrainer.ALGORITHM_PARAM, GIS.MAXENT_VALUE);
trainParams.put(AbstractEventTrainer.DATA_INDEXER_PARAM,
AbstractEventTrainer.DATA_INDEXER_TWO_PASS_VALUE);
trainParams.put(AbstractTrainer.CUTOFF_PARAM, Integer.toString(1));
-
+
MaxentModel model = TrainUtil.train(createTrainingStream(), trainParams, null);
-
+
testModel(model, 0.7997028967566229);
}
-
+
@Test
public void testMaxentOnPrepAttachDataWithParamsDefault() throws IOException {
-
+
Map<String, String> trainParams = new HashMap<String, String>();
trainParams.put(AbstractTrainer.ALGORITHM_PARAM, GIS.MAXENT_VALUE);
-
+
MaxentModel model = TrainUtil.train(createTrainingStream(), trainParams, null);
-
+
testModel(model, 0.8086159940579352 );
}
-
+
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/RealValueModelTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/RealValueModelTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/RealValueModelTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/RealValueModelTest.java Fri May 2 12:34:23 2014
@@ -63,7 +63,7 @@ public class RealValueModelTest extends
for(int i=0; i<realResults.length; i++) {
System.out.println(String.format("classifiy with realModel: %1$s = %2$f", realModel.getOutcome(i), realResults[i]));
System.out.println(String.format("classifiy with repeatModel: %1$s = %2$f", repeatModel.getOutcome(i), repeatResults[i]));
- assertEquals(realResults[i], repeatResults[i], 0.01f);
+ assertEquals(realResults[i], repeatResults[i], 0.01f);
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/ScaleDoesntMatterTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/ScaleDoesntMatterTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/ScaleDoesntMatterTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/maxent/ScaleDoesntMatterTest.java Fri May 2 12:34:23 2014
@@ -34,7 +34,7 @@ public class ScaleDoesntMatterTest exten
* predicates doesn't matter when it comes the probability assigned to each
* outcome. Strangely, if we use (1,2) and (10,20) there's no difference. If
* we use (0.1,0.2) and (10,20) there is a difference.
- *
+ *
* @throws Exception
*/
public void testScaleResults() throws Exception {
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/perceptron/PerceptronPrepAttachTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/perceptron/PerceptronPrepAttachTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/perceptron/PerceptronPrepAttachTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/ml/perceptron/PerceptronPrepAttachTest.java Fri May 2 12:34:23 2014
@@ -38,51 +38,51 @@ public class PerceptronPrepAttachTest {
@Test
public void testPerceptronOnPrepAttachData() throws IOException {
- MaxentModel model =
- new PerceptronTrainer().trainModel(400,
+ MaxentModel model =
+ new PerceptronTrainer().trainModel(400,
new TwoPassDataIndexer(createTrainingStream(), 1, false), 1);
testModel(model, 0.7650408516959644);
}
-
+
@Test
public void testPerceptronOnPrepAttachDataWithSkippedAveraging() throws IOException {
-
+
Map<String, String> trainParams = new HashMap<String, String>();
trainParams.put(AbstractTrainer.ALGORITHM_PARAM, PerceptronTrainer.PERCEPTRON_VALUE);
trainParams.put(AbstractTrainer.CUTOFF_PARAM, Integer.toString(1));
trainParams.put("UseSkippedAveraging", Boolean.toString(true));
-
+
MaxentModel model = TrainUtil.train(createTrainingStream(), trainParams, null);
-
+
testModel(model, 0.773706362961129);
}
-
+
@Test
public void testPerceptronOnPrepAttachDataWithTolerance() throws IOException {
-
+
Map<String, String> trainParams = new HashMap<String, String>();
trainParams.put(AbstractTrainer.ALGORITHM_PARAM, PerceptronTrainer.PERCEPTRON_VALUE);
trainParams.put(AbstractTrainer.CUTOFF_PARAM, Integer.toString(1));
trainParams.put(AbstractTrainer.ITERATIONS_PARAM, Integer.toString(500));
trainParams.put("Tolerance", Double.toString(0.0001d));
-
+
MaxentModel model = TrainUtil.train(createTrainingStream(), trainParams, null);
-
+
testModel(model, 0.7677642980935875);
}
-
+
@Test
public void testPerceptronOnPrepAttachDataWithStepSizeDecrease() throws IOException {
-
+
Map<String, String> trainParams = new HashMap<String, String>();
trainParams.put(AbstractTrainer.ALGORITHM_PARAM, PerceptronTrainer.PERCEPTRON_VALUE);
trainParams.put(AbstractTrainer.CUTOFF_PARAM, Integer.toString(1));
trainParams.put(AbstractTrainer.ITERATIONS_PARAM, Integer.toString(500));
trainParams.put("StepSizeDecrease", Double.toString(0.06d));
-
+
MaxentModel model = TrainUtil.train(createTrainingStream(), trainParams, null);
-
+
testModel(model, 0.7756870512503095);
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java Fri May 2 12:34:23 2014
@@ -59,7 +59,7 @@ public class DictionaryNameFinderEvaluat
/**
* Creates a NameSample stream using an annotated corpus
- *
+ *
* @return
* @throws IOException
* @throws URISyntaxException
@@ -77,7 +77,7 @@ public class DictionaryNameFinderEvaluat
/**
* Creates a dictionary with all names from the sample data.
- *
+ *
* @return a dictionary
* @throws IOException
* @throws URISyntaxException
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderTest.java Fri May 2 12:34:23 2014
@@ -47,7 +47,7 @@ public class DictionaryNameFinderTest{
StringList max = new StringList(new String[]{"Max"});
mDictionary.put(max);
-
+
StringList michaelJordan = new
StringList(new String[]{"Michael", "Jordan"});
mDictionary.put(michaelJordan);
@@ -127,13 +127,13 @@ public class DictionaryNameFinderTest{
assertTrue(names.length == 1);
assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 5);
}
-
+
@Test
public void testCaseLongerEntry() {
String sentence[] = {"a", "b", "michael", "jordan"};
-
+
Span names[] = mNameFinder.find(sentence);
-
+
assertTrue(names.length == 1);
assertTrue(names[0].length() == 2);
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderEventStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderEventStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderEventStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderEventStreamTest.java Fri May 2 12:34:23 2014
@@ -52,20 +52,20 @@ public class NameFinderEventStreamTest{
"traditional",
"meal",
"."};
-
- NameSample nameSample = new NameSample(sentence,
+
+ NameSample nameSample = new NameSample(sentence,
new Span[]{new Span(0, 2, "person")}, false);
-
+
ObjectStream<Event> eventStream = new NameFinderEventStream(
ObjectStreamUtils.createObjectStream(nameSample));
-
+
assertEquals("person-" + NameFinderME.START, eventStream.read().getOutcome());
assertEquals("person-" + NameFinderME.CONTINUE, eventStream.read().getOutcome());
-
+
for (int i = 0; i < 10; i++) {
Assert.assertEquals(NameFinderME.OTHER, eventStream.read().getOutcome());
}
-
+
assertNull(eventStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java Fri May 2 12:34:23 2014
@@ -72,7 +72,7 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
@@ -135,10 +135,10 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
-
+
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
// now test if it can detect the sample sentences
@@ -184,10 +184,10 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
-
+
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
// now test if it can detect the sample sentences
@@ -223,10 +223,10 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
-
+
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
// now test if it can detect the sample sentences
@@ -264,7 +264,7 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
@@ -321,7 +321,7 @@ public class NameFinderMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
params, (byte[]) null, Collections.<String, Object>emptyMap());
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleDataStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleDataStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleDataStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleDataStreamTest.java Fri May 2 12:34:23 2014
@@ -44,7 +44,7 @@ import org.junit.Test;
* This is the test class for {@link NameSampleDataStream}..
*/
public class NameSampleDataStreamTest {
-
+
final String person = "person";
final String date = "date";
final String location = "location";
@@ -52,7 +52,7 @@ public class NameSampleDataStreamTest {
/**
* Create a string from a array section.
- *
+ *
* @param tokens the tokens
* @param nameSpan the section
* @return the string
@@ -65,11 +65,11 @@ public class NameSampleDataStreamTest {
return sb.toString().trim();
}
-
+
/**
* Create a NameSampleDataStream from a corpus with entities annotated but
* without nameType and validate it.
- *
+ *
* @throws Exception
*/
@Test
@@ -85,10 +85,10 @@ public class NameSampleDataStreamTest {
NameSample ns = ds.read();
String[] expectedNames = { "Alan McKennedy", "Julie", "Marie Clara",
- "Stefanie Schmidt", "Mike", "Stefanie Schmidt", "George", "Luise",
- "George Bauer", "Alisa Fernandes", "Alisa", "Mike Sander",
- "Stefan Miller", "Stefan Miller", "Stefan Miller", "Elenor Meier",
- "Gina Schneider", "Bruno Schulz", "Michel Seile", "George Miller",
+ "Stefanie Schmidt", "Mike", "Stefanie Schmidt", "George", "Luise",
+ "George Bauer", "Alisa Fernandes", "Alisa", "Mike Sander",
+ "Stefan Miller", "Stefan Miller", "Stefan Miller", "Elenor Meier",
+ "Gina Schneider", "Bruno Schulz", "Michel Seile", "George Miller",
"Miller", "Peter Schubert", "Natalie" };
List<String> names = new ArrayList<String>();
@@ -127,7 +127,7 @@ public class NameSampleDataStreamTest {
assertEquals(createDefaultSpan(2,4), spans.get(21));
assertEquals(createDefaultSpan(5,6), spans.get(22));
}
-
+
private Span createDefaultSpan(int s, int e) {
return new Span(s, e, NameSample.DEFAULT_TYPE);
}
@@ -139,36 +139,36 @@ public class NameSampleDataStreamTest {
public void testWithoutNameTypeAndInvalidData() {
NameSampleDataStream sampleStream = new NameSampleDataStream(
ObjectStreamUtils.createObjectStream("<START> <START> Name <END>"));
-
+
try {
sampleStream.read();
fail();
} catch (IOException e) {
}
-
+
sampleStream = new NameSampleDataStream(
ObjectStreamUtils.createObjectStream("<START> Name <END> <END>"));
-
+
try {
sampleStream.read();
fail();
} catch (IOException e) {
}
-
+
sampleStream = new NameSampleDataStream(
ObjectStreamUtils.createObjectStream("<START> <START> Person <END> Street <END>"));
-
+
try {
sampleStream.read();
fail();
} catch (IOException e) {
}
}
-
+
/**
* Create a NameSampleDataStream from a corpus with entities annotated
* with multiple nameTypes, like person, date, location and organization, and validate it.
- *
+ *
* @throws Exception
*/
@Test
@@ -181,7 +181,7 @@ public class NameSampleDataStreamTest {
Map<String, List<String>> names = new HashMap<String, List<String>>();
Map<String, List<Span>> spans = new HashMap<String, List<Span>>();
-
+
NameSample ns;
while ((ns = ds.read()) != null) {
Span[] nameSpans = ns.getNames();
@@ -197,7 +197,7 @@ public class NameSampleDataStreamTest {
.add(nameSpan);
}
}
-
+
String[] expectedPerson = { "Barack Obama", "Obama", "Obama",
"Lee Myung - bak", "Obama", "Obama", "Scott Snyder", "Snyder", "Obama",
"Obama", "Obama", "Tim Peters", "Obama", "Peters" };
@@ -208,14 +208,14 @@ public class NameSampleDataStreamTest {
"China", "South Korea", "North Korea", "North Korea", "U . S .",
"South Korea", "United States", "Pyongyang", "North Korea",
"South Korea", "Afghanistan", "Seoul", "U . S .", "China" };
-
+
String[] expectedOrganization = {"Center for U . S . Korea Policy"};
-
+
assertEquals(expectedPerson.length, names.get(person).size());
assertEquals(expectedDate.length, names.get(date).size());
assertEquals(expectedLocation.length, names.get(location).size());
assertEquals(expectedOrganization.length, names.get(organization).size());
-
+
assertEquals(new Span(5,7, person), spans.get(person).get(0));
assertEquals(expectedPerson[0], names.get(person).get(0));
assertEquals(new Span(10,11, person), spans.get(person).get(1));
@@ -251,7 +251,7 @@ public class NameSampleDataStreamTest {
assertEquals(expectedDate[1], names.get(date).get(1));
assertEquals(new Span(15,16, date), spans.get(date).get(2));
assertEquals(expectedDate[2], names.get(date).get(2));
-
+
assertEquals(new Span(0, 4, location), spans.get(location).get(0));
assertEquals(expectedLocation[0], names.get(location).get(0));
assertEquals(new Span(10,12, location), spans.get(location).get(1));
@@ -286,34 +286,34 @@ public class NameSampleDataStreamTest {
assertEquals(expectedLocation[15], names.get(location).get(15));
assertEquals(new Span(11,12, location), spans.get(location).get(16));
assertEquals(expectedLocation[16], names.get(location).get(16));
-
+
assertEquals(new Span(7,15, organization), spans.get(organization).get(0));
assertEquals(expectedOrganization[0], names.get(organization).get(0));
-
+
}
-
+
@Test
public void testWithNameTypeAndInvalidData() {
-
+
NameSampleDataStream sampleStream = new NameSampleDataStream(
ObjectStreamUtils.createObjectStream("<START:> Name <END>"));
-
+
try {
sampleStream.read();
fail();
} catch (IOException e) {
}
-
+
sampleStream = new NameSampleDataStream(
ObjectStreamUtils.createObjectStream("<START:street> <START:person> Name <END> <END>"));
-
+
try {
sampleStream.read();
fail();
} catch (IOException e) {
}
}
-
+
@Test
public void testClearAdaptiveData() throws IOException {
StringBuilder trainingData = new StringBuilder();
@@ -322,36 +322,36 @@ public class NameSampleDataStreamTest {
trainingData.append("c\n");
trainingData.append("\n");
trainingData.append("d\n");
-
+
ObjectStream<String> untokenizedLineStream =
new PlainTextByLineStream(new StringReader(trainingData.toString()));
-
+
ObjectStream<NameSample> trainingStream = new NameSampleDataStream(untokenizedLineStream);
-
+
assertFalse(trainingStream.read().isClearAdaptiveDataSet());
assertFalse(trainingStream.read().isClearAdaptiveDataSet());
assertFalse(trainingStream.read().isClearAdaptiveDataSet());
assertTrue(trainingStream.read().isClearAdaptiveDataSet());
assertNull(trainingStream.read());
}
-
+
@Test
public void testHtmlNameSampleParsing() throws IOException {
InputStream in = getClass().getClassLoader().getResourceAsStream(
"opennlp/tools/namefind/html1.train");
-
+
NameSampleDataStream ds = new NameSampleDataStream(
new PlainTextByLineStream(new InputStreamReader(in, "UTF-8")));
NameSample ns = ds.read();
-
+
assertEquals(1, ns.getSentence().length);
assertEquals("<html>", ns.getSentence()[0]);
-
+
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("<head/>", ns.getSentence()[0]);
-
+
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("<body>", ns.getSentence()[0]);
@@ -359,7 +359,7 @@ public class NameSampleDataStreamTest {
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("<ul>", ns.getSentence()[0]);
-
+
// <li> <START:organization> Advanced Integrated Pest Management <END> </li>
ns = ds.read();
assertEquals(6, ns.getSentence().length);
@@ -370,7 +370,7 @@ public class NameSampleDataStreamTest {
assertEquals("Management", ns.getSentence()[4]);
assertEquals("</li>", ns.getSentence()[5]);
assertEquals(new Span(1, 5, organization), ns.getNames()[0]);
-
+
// <li> <START:organization> Bay Cities Produce Co., Inc. <END> </li>
ns = ds.read();
assertEquals(7, ns.getSentence().length);
@@ -382,19 +382,19 @@ public class NameSampleDataStreamTest {
assertEquals("Inc.", ns.getSentence()[5]);
assertEquals("</li>", ns.getSentence()[6]);
assertEquals(new Span(1, 6, organization), ns.getNames()[0]);
-
+
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("</ul>", ns.getSentence()[0]);
-
+
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("</body>", ns.getSentence()[0]);
-
+
ns = ds.read();
assertEquals(1, ns.getSentence().length);
assertEquals("</html>", ns.getSentence()[0]);
-
+
assertNull(ds.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/NameSampleTest.java Fri May 2 12:34:23 2014
@@ -32,22 +32,22 @@ import org.junit.Test;
* This is the test class for {@link NameSample}.
*/
public class NameSampleTest {
-
+
/**
* Create a NameSample from scratch and validate it.
- *
+ *
* @param useTypes if to use nametypes
* @return the NameSample
*/
private static NameSample createSimpleNameSample(boolean useTypes) {
-
+
String[] sentence = {"U", ".", "S", ".", "President", "Barack", "Obama", "is",
"considering", "sending", "additional", "American", "forces",
"to", "Afghanistan", "."};
-
- Span[] names = {new Span(0, 4, "Location"), new Span(5, 7, "Person"),
+
+ Span[] names = {new Span(0, 4, "Location"), new Span(5, 7, "Person"),
new Span(14, 15, "Location")};
-
+
NameSample nameSample;
if(useTypes) {
nameSample = new NameSample(sentence, names, false);
@@ -55,16 +55,16 @@ public class NameSampleTest {
else {
Span[] namesWithoutType = new Span[names.length];
for (int i = 0; i < names.length; i++) {
- namesWithoutType[i] = new Span(names[i].getStart(),
+ namesWithoutType[i] = new Span(names[i].getStart(),
names[i].getEnd());
}
-
+
nameSample = new NameSample(sentence, namesWithoutType, false);
}
-
+
return nameSample;
}
-
+
/**
* Checks if could create a NameSample without NameTypes, generate the
* string representation and validate it.
@@ -72,11 +72,11 @@ public class NameSampleTest {
@Test
public void testNoTypesToString() {
String nameSampleStr = createSimpleNameSample(false).toString();
-
+
assertEquals("<START> U . S . <END> President <START> Barack Obama <END> is considering " +
"sending additional American forces to <START> Afghanistan <END> .", nameSampleStr);
}
-
+
/**
* Checks if could create a NameSample with NameTypes, generate the
* string representation and validate it.
@@ -85,37 +85,37 @@ public class NameSampleTest {
public void testWithTypesToString() throws Exception {
String nameSampleStr = createSimpleNameSample(true).toString();
assertEquals("<START:Location> U . S . <END> President <START:Person> Barack Obama <END> is considering sending additional American forces to <START:Location> Afghanistan <END> .", nameSampleStr);
-
+
NameSample parsedSample = NameSample.parse("<START:Location> U . S . <END> " +
"President <START:Person> Barack Obama <END> is considering sending " +
- "additional American forces to <START:Location> Afghanistan <END> .",
+ "additional American forces to <START:Location> Afghanistan <END> .",
false);
-
+
assertEquals(createSimpleNameSample(true), parsedSample);
}
-
+
/**
* Checks that if the name is the last token in a sentence it is still outputed
* correctly.
*/
@Test
public void testNameAtEnd() {
-
+
String sentence[] = new String[] {
"My",
"name",
"is",
"Anna"
};
-
+
NameSample sample = new NameSample(sentence, new Span[]{new Span(3, 4)}, false);
-
+
assertEquals("My name is <START> Anna <END>", sample.toString());
}
-
+
/**
* Tests if an additional space is correctly treated as one space.
- *
+ *
* @throws Exception
*/
@Test
@@ -123,10 +123,10 @@ public class NameSampleTest {
String line = "<START> M . K . <END> <START> Schwitters <END> ? <START> Heartfield <END> ?";
NameSample test = NameSample.parse(line, false);
-
+
assertEquals(8, test.getSentence().length);
}
-
+
/**
* Checks if it accepts name type with some special characters
*/
@@ -144,23 +144,23 @@ public class NameSampleTest {
assertEquals("type_2", parsedSample.getNames()[1].getType());
assertEquals("type_3-/;.,&%$", parsedSample.getNames()[2].getType());
}
-
+
/**
* Test if it fails to parse empty type
*/
@Test(expected=IOException.class)
public void testMissingType() throws Exception {
- NameSample.parse("<START:> token <END>",
+ NameSample.parse("<START:> token <END>",
false);
}
-
+
/**
* Test if it fails to parse type with space
* @throws Exception
*/
@Test(expected=IOException.class)
public void testTypeWithSpace() throws Exception {
- NameSample.parse("<START:abc a> token <END>",
+ NameSample.parse("<START:abc a> token <END>",
false);
}
@@ -170,7 +170,7 @@ public class NameSampleTest {
*/
@Test(expected=IOException.class)
public void testTypeWithNewLine() throws Exception {
- NameSample.parse("<START:abc\na> token <END>",
+ NameSample.parse("<START:abc\na> token <END>",
false);
}
@@ -180,20 +180,20 @@ public class NameSampleTest {
*/
@Test(expected=IOException.class)
public void testTypeWithInvalidChar1() throws Exception {
- NameSample.parse("<START:abc:a> token <END>",
+ NameSample.parse("<START:abc:a> token <END>",
false);
}
-
+
/**
* Test if it fails to parse type with >
* @throws Exception
*/
@Test(expected=IOException.class)
public void testTypeWithInvalidChar2() throws Exception {
- NameSample.parse("<START:abc>a> token <END>",
+ NameSample.parse("<START:abc>a> token <END>",
false);
}
-
+
@Test
public void testEquals() {
assertFalse(createGoldSample() == createGoldSample());
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java Fri May 2 12:34:23 2014
@@ -54,7 +54,7 @@ public class TokenNameFinderCrossValidat
TrainingParameters mlParams = new TrainingParameters();
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
mlParams.put(TrainingParameters.ALGORITHM_PARAM,
ModelType.MAXENT.toString());
@@ -65,7 +65,7 @@ public class TokenNameFinderCrossValidat
assertNotNull(cv.getFMeasure());
}
-
+
@Test
/**
* Test that tries to reproduce jira OPENNLP-466
@@ -82,19 +82,19 @@ public class TokenNameFinderCrossValidat
TrainingParameters mlParams = new TrainingParameters();
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
mlParams.put(TrainingParameters.ALGORITHM_PARAM,
ModelType.MAXENT.toString());
-
+
ByteArrayOutputStream out = new ByteArrayOutputStream();
- NameEvaluationErrorListener listener = new NameEvaluationErrorListener(out);
+ NameEvaluationErrorListener listener = new NameEvaluationErrorListener(out);
Map<String, Object> resources = Collections.emptyMap();
TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("en",
TYPE, mlParams, null, resources, listener);
cv.evaluate(sampleStream, 2);
-
+
assertTrue(out.size() > 0);
assertNotNull(cv.getFMeasure());
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java Fri May 2 12:34:23 2014
@@ -38,33 +38,33 @@ public class TokenNameFinderEvaluatorTes
public void testPositive() {
OutputStream stream = new ByteArrayOutputStream();
TokenNameFinderEvaluationMonitor listener = new NameEvaluationErrorListener(stream);
-
+
Span[] pred = createSimpleNameSampleA().getNames();
TokenNameFinderEvaluator eval = new TokenNameFinderEvaluator(new DummyNameFinder(pred), listener);
-
+
eval.evaluateSample(createSimpleNameSampleA());
-
+
assertEquals(1.0, eval.getFMeasure().getFMeasure());
-
+
assertEquals(0, stream.toString().length());
}
-
+
@Test
public void testNegative() {
OutputStream stream = new ByteArrayOutputStream();
TokenNameFinderEvaluationMonitor listener = new NameEvaluationErrorListener(stream);
-
+
Span[] pred = createSimpleNameSampleB().getNames();
TokenNameFinderEvaluator eval = new TokenNameFinderEvaluator(new DummyNameFinder(pred), listener);
-
+
eval.evaluateSample(createSimpleNameSampleA());
-
+
assertEquals(0.8, eval.getFMeasure().getFMeasure());
-
+
assertNotSame(0, stream.toString().length());
}
-
-
+
+
private static String[] sentence = {"U", ".", "S", ".", "President", "Barack", "Obama", "is",
"considering", "sending", "additional", "American", "forces",
@@ -90,10 +90,10 @@ public class TokenNameFinderEvaluatorTes
return nameSample;
}
-
+
/** a dummy name finder that always return something expected */
class DummyNameFinder implements TokenNameFinder {
-
+
private Span[] ret;
public DummyNameFinder(Span[] ret) {
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ChunkSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ChunkSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ChunkSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ChunkSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -34,9 +34,9 @@ public class ChunkSampleStreamTest {
public void testConvertParseToPosSample() throws IOException {
ObjectStream<ChunkSample> chunkSampleStream = new ChunkSampleStream(new ParseSampleStream(
ObjectStreamUtils.createObjectStream(ParseTest.PARSE_STRING)));
-
+
ChunkSample sample = chunkSampleStream.read();
-
+
assertEquals("She", sample.getSentence()[0]);
assertEquals("PRP", sample.getTags()[0]);
assertEquals("S-NP", sample.getPreds()[0]);
@@ -91,7 +91,7 @@ public class ChunkSampleStreamTest {
assertEquals(".", sample.getSentence()[17]);
assertEquals(".", sample.getTags()[17]);
assertEquals("O", sample.getPreds()[17]);
-
+
assertNull(chunkSampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -32,17 +32,17 @@ import org.junit.Test;
public class ParseSampleStreamTest {
static ObjectStream<Parse> createParseSampleStream() throws IOException {
-
+
InputStream in = ParseSampleStreamTest.class.getResourceAsStream(
"/opennlp/tools/parser/test.parse");
-
+
return new ParseSampleStream(new PlainTextByLineStream(new InputStreamReader(in, "UTF-8")));
}
-
+
@Test
public void testReadTestStream() throws IOException {
ObjectStream<Parse> parseStream = createParseSampleStream();
-
+
assertNotNull(parseStream.read());
assertNotNull(parseStream.read());
assertNotNull(parseStream.read());
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParseTest.java Fri May 2 12:34:23 2014
@@ -28,25 +28,25 @@ import org.junit.Test;
public class ParseTest {
public static final String PARSE_STRING = "(TOP (S (S (NP-SBJ (PRP She) )(VP (VBD was) (ADVP (RB just) )(NP-PRD (NP (DT another) (NN freighter) )(PP (IN from) (NP (DT the) (NNPS States) )))))(, ,) (CC and) (S (NP-SBJ (PRP she) )(VP (VBD seemed) (ADJP-PRD (ADJP (RB as) (JJ commonplace) )(PP (IN as) (NP (PRP$ her) (NN name) )))))(. .) ))";
-
+
@Test
public void testToHashCode() {
Parse p1 = Parse.parseParse(PARSE_STRING);
p1.hashCode();
}
-
+
@Test
public void testToString() {
Parse p1 = Parse.parseParse(PARSE_STRING);
p1.toString();
}
-
+
@Test
public void testEquals() {
Parse p1 = Parse.parseParse(PARSE_STRING);
assertTrue(p1.equals(p1));
}
-
+
@Test
public void testParseClone() {
Parse p1 = Parse.parseParse(PARSE_STRING);
@@ -54,26 +54,26 @@ public class ParseTest {
assertTrue(p1.equals(p2));
assertTrue(p2.equals(p1));
}
-
+
@Test
public void testGetText() {
Parse p = Parse.parseParse(PARSE_STRING);
-
+
// TODO: Why does parse attaches a space to the end of the text ???
String expectedText = "She was just another freighter from the States , and she seemed as commonplace as her name . ";
-
+
assertEquals(expectedText, p.getText());
}
-
+
@Test
public void testShow() {
Parse p1 = Parse.parseParse(PARSE_STRING);
-
+
StringBuffer parseString = new StringBuffer();
p1.show(parseString);
-
+
Parse p2 = Parse.parseParse(parseString.toString());
-
+
assertEquals(p1, p2);
}
@@ -89,25 +89,25 @@ public class ParseTest {
" )))(SBAR (WHNP-1 (WDT that) )(S (VP (VBD put) " +
" (NP (DT the) (NN spotlight) )(PP (IN on) (NP (DT the) " +
" (JJ international) (NN play-girl) ))))))(. .) ))");
-
+
StringBuffer parseString = new StringBuffer();
p1.show(parseString);
-
+
Parse p2 = Parse.parseParse(parseString.toString());
-
+
assertEquals(p1, p2);
}
-
+
@Test
public void testGetTagNodes() {
Parse p = Parse.parseParse(PARSE_STRING);
-
+
Parse tags[] = p.getTagNodes();
-
+
for (Parse node : tags) {
assertTrue(node.isPosTag());
}
-
+
assertEquals("PRP", tags[0].getType());
assertEquals("VBD", tags[1].getType());
assertEquals("RB", tags[2].getType());
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParserTestUtil.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParserTestUtil.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParserTestUtil.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/ParserTestUtil.java Fri May 2 12:34:23 2014
@@ -31,24 +31,24 @@ import opennlp.tools.util.PlainTextByLin
public class ParserTestUtil {
public static HeadRules createTestHeadRules() throws IOException {
- InputStream headRulesIn =
+ InputStream headRulesIn =
ParserTestUtil.class.getResourceAsStream("/opennlp/tools/parser/en_head_rules");
-
+
HeadRules headRules = new HeadRules(new BufferedReader(
new InputStreamReader(headRulesIn, "UTF-8")));
-
+
headRulesIn.close();
-
+
return headRules;
}
-
- public static ObjectStream<Parse> openTestTrainingData()
+
+ public static ObjectStream<Parse> openTestTrainingData()
throws IOException {
-
+
ObjectStream<Parse> resetableSampleStream = new ObjectStream<Parse> () {
-
+
private ObjectStream<Parse> samples;
-
+
public void close() throws IOException {
samples.close();
}
@@ -61,7 +61,7 @@ public class ParserTestUtil {
try {
if (samples != null)
samples.close();
-
+
samples = new ParseSampleStream(new PlainTextByLineStream(
new InputStreamReader(
ParserTestUtil.class.getResourceAsStream("/opennlp/tools/parser/parser.train"), "UTF-8")));
@@ -71,9 +71,9 @@ public class ParserTestUtil {
}
}
};
-
+
resetableSampleStream.reset();
-
+
return resetableSampleStream;
- }
+ }
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/PosSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/PosSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/PosSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/parser/PosSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -29,15 +29,15 @@ import opennlp.tools.util.ObjectStreamUt
import org.junit.Test;
public class PosSampleStreamTest {
-
+
@Test
public void testConvertParseToPosSample() throws IOException {
-
+
ObjectStream<POSSample> posSampleStream = new PosSampleStream(new ParseSampleStream(
ObjectStreamUtils.createObjectStream(ParseTest.PARSE_STRING)));
-
+
POSSample sample = posSampleStream.read();
-
+
assertEquals("PRP", sample.getTags()[0]);
assertEquals("She", sample.getSentence()[0]);
assertEquals("VBD", sample.getTags()[1]);
@@ -74,7 +74,7 @@ public class PosSampleStreamTest {
assertEquals("name", sample.getSentence()[16]);
assertEquals(".", sample.getTags()[17]);
assertEquals(".", sample.getSentence()[17]);
-
+
assertNull(posSampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/DummyPOSTaggerFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/DummyPOSTaggerFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/DummyPOSTaggerFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/DummyPOSTaggerFactory.java Fri May 2 12:34:23 2014
@@ -29,11 +29,11 @@ import opennlp.tools.util.model.Artifact
import opennlp.tools.util.model.UncloseableInputStream;
public class DummyPOSTaggerFactory extends POSTaggerFactory {
-
+
private static final String DUMMY_POSDICT = "DUMMY_POSDICT";
private DummyPOSDictionary dict;
-
+
public DummyPOSTaggerFactory() {
}
@@ -41,31 +41,31 @@ public class DummyPOSTaggerFactory exten
super(ngramDictionary, null);
this.dict = posDictionary;
}
-
+
@Override
public SequenceValidator<String> getSequenceValidator() {
return new DummyPOSSequenceValidator();
}
-
+
@Override
public DummyPOSDictionary getTagDictionary() {
return (DummyPOSDictionary) artifactProvider.getArtifact(DUMMY_POSDICT);
}
-
+
@Override
public POSContextGenerator getPOSContextGenerator() {
return new DummyPOSContextGenerator(this.ngramDictionary);
}
-
+
@Override
@SuppressWarnings("rawtypes")
public Map<String, ArtifactSerializer> createArtifactSerializersMap() {
Map<String, ArtifactSerializer> serializers = super.createArtifactSerializersMap();
-
+
serializers.put(DUMMY_POSDICT, new DummyPOSDictionarySerializer());
return serializers;
}
-
+
@Override
public Map<String, Object> createArtifactMap() {
Map<String, Object> artifactMap = super.createArtifactMap();
@@ -73,15 +73,15 @@ public class DummyPOSTaggerFactory exten
artifactMap.put(DUMMY_POSDICT, this.dict);
return artifactMap;
}
-
+
static class DummyPOSContextGenerator extends DefaultPOSContextGenerator {
public DummyPOSContextGenerator(Dictionary dict) {
super(dict);
}
-
+
}
-
+
static class DummyPOSDictionarySerializer implements ArtifactSerializer<DummyPOSDictionary> {
public DummyPOSDictionary create(InputStream in) throws IOException,
@@ -94,20 +94,20 @@ public class DummyPOSTaggerFactory exten
artifact.serialize(out);
}
}
-
+
static class DummyPOSSequenceValidator implements SequenceValidator<String> {
public boolean validSequence(int i, String[] inputSequence,
String[] outcomesSequence, String outcome) {
return true;
}
-
+
}
-
+
static class DummyPOSDictionary extends POSDictionary {
- private POSDictionary dict;
-
+ private POSDictionary dict;
+
public DummyPOSDictionary(POSDictionary dict) {
this.dict = dict;
}
@@ -126,5 +126,5 @@ public class DummyPOSTaggerFactory exten
}
}
-
+
}
\ No newline at end of file
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSDictionaryTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSDictionaryTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSDictionaryTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSDictionaryTest.java Fri May 2 12:34:23 2014
@@ -39,7 +39,7 @@ public class POSDictionaryTest {
private static POSDictionary loadDictionary(String name) throws IOException {
return POSDictionary.create(POSDictionaryTest.class.getResourceAsStream(name));
}
-
+
private static POSDictionary serializeDeserializeDict(POSDictionary dict) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -59,10 +59,10 @@ public class POSDictionaryTest {
finally {
in.close();
}
-
+
return serializedDictionary;
}
-
+
@Test
public void testSerialization() throws IOException, InvalidFormatException {
POSDictionary dictionary = new POSDictionary();
@@ -74,11 +74,11 @@ public class POSDictionaryTest {
assertTrue(dictionary.equals(serializeDeserializeDict(dictionary)));
}
-
+
@Test
public void testLoadingDictionaryWithoutCaseAttribute() throws IOException {
POSDictionary dict = loadDictionary("TagDictionaryWithoutCaseAttribute.xml");
-
+
assertArrayEquals(new String[]{"NNP"}, dict.getTags("McKinsey"));
assertNull(dict.getTags("Mckinsey"));
}
@@ -89,28 +89,28 @@ public class POSDictionaryTest {
assertArrayEquals(new String[]{"NNP"}, dict.getTags("McKinsey"));
assertNull(dict.getTags("Mckinsey"));
-
+
dict = serializeDeserializeDict(dict);
-
+
assertArrayEquals(new String[]{"NNP"}, dict.getTags("McKinsey"));
assertNull(dict.getTags("Mckinsey"));
}
-
+
@Test
public void testCaseInsensitiveDictionary() throws IOException {
POSDictionary dict = loadDictionary("TagDictionaryCaseInsensitive.xml");
-
+
assertArrayEquals(new String[]{"NNP"}, dict.getTags("McKinsey"));
assertArrayEquals(new String[]{"NNP"}, dict.getTags("Mckinsey"));
assertArrayEquals(new String[]{"NNP"}, dict.getTags("MCKINSEY"));
assertArrayEquals(new String[]{"NNP"}, dict.getTags("mckinsey"));
-
+
dict = serializeDeserializeDict(dict);
-
+
assertArrayEquals(new String[]{"NNP"}, dict.getTags("McKinsey"));
assertArrayEquals(new String[]{"NNP"}, dict.getTags("Mckinsey"));
}
-
+
@Test
public void testToString() throws IOException {
POSDictionary dict = loadDictionary("TagDictionaryCaseInsensitive.xml");
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSEvaluatorTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSEvaluatorTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSEvaluatorTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSEvaluatorTest.java Fri May 2 12:34:23 2014
@@ -33,41 +33,41 @@ import org.junit.Test;
public class POSEvaluatorTest {
-
+
@Test
public void testPositive() throws InvalidFormatException {
OutputStream stream = new ByteArrayOutputStream();
POSTaggerEvaluationMonitor listener = new POSEvaluationErrorListener(stream);
-
+
POSEvaluator eval = new POSEvaluator(new DummyPOSTagger(
POSSampleTest.createGoldSample()), listener);
-
+
eval.evaluateSample(POSSampleTest.createGoldSample());
-
+
assertEquals(1.0, eval.getWordAccuracy());
-
+
assertEquals(0, stream.toString().length());
}
-
+
@Test
public void testNegative() throws InvalidFormatException {
OutputStream stream = new ByteArrayOutputStream();
POSTaggerEvaluationMonitor listener = new POSEvaluationErrorListener(stream);
-
+
POSEvaluator eval = new POSEvaluator(new DummyPOSTagger(POSSampleTest.createGoldSample()), listener);
-
+
eval.evaluateSample(POSSampleTest.createPredSample());
-
+
assertEquals(.7, eval.getWordAccuracy(), .1d);
-
+
assertNotSame(0, stream.toString().length());
}
-
+
class DummyPOSTagger implements POSTagger {
-
+
private POSSample sample;
-
+
public DummyPOSTagger(POSSample sample) {
this.sample = sample;
}
@@ -99,7 +99,7 @@ public class POSEvaluatorTest {
public Sequence[] topKSequences(String[] sentence, Object[] additionaContext) {
return topKSequences(sentence);
}
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSModelTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSModelTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSModelTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSModelTest.java Fri May 2 12:34:23 2014
@@ -46,22 +46,22 @@ public class POSModelTest {
// TODO: add equals to pos model
}
-
+
@Test
public void testPOSModelSerializationPerceptron() throws IOException, InvalidFormatException {
POSModel posModel = POSTaggerMETest.trainPOSModel(ModelType.PERCEPTRON);
-
+
ByteArrayOutputStream out = new ByteArrayOutputStream();
-
+
try {
posModel.serialize(out);
}
finally {
out.close();
}
-
+
POSModel recreatedPosModel = new POSModel(new ByteArrayInputStream(out.toByteArray()));
-
+
// TODO: add equals to pos model
}
}
\ No newline at end of file
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleEventStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleEventStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleEventStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleEventStreamTest.java Fri May 2 12:34:23 2014
@@ -31,20 +31,20 @@ import org.junit.Test;
public class POSSampleEventStreamTest {
/**
- * Tests that the outcomes for a single sentence match the
+ * Tests that the outcomes for a single sentence match the
* expected outcomes.
- *
+ *
* @throws Exception
*/
@Test
public void testOutcomesForSingleSentence() throws Exception {
String sentence = "That_DT sounds_VBZ good_JJ ._.";
-
+
POSSample sample = POSSample.parse(sentence);
-
+
ObjectStream<Event> eventStream = new POSSampleEventStream(
ObjectStreamUtils.createObjectStream(sample));
-
+
Assert.assertEquals("DT", eventStream.read().getOutcome());
Assert.assertEquals("VBZ", eventStream.read().getOutcome());
Assert.assertEquals("JJ", eventStream.read().getOutcome());
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSSampleTest.java Fri May 2 12:34:23 2014
@@ -30,7 +30,7 @@ import org.junit.Test;
* Tests for the {@link POSSample} class.
*/
public class POSSampleTest {
-
+
@Test
public void testEquals() throws InvalidFormatException {
assertFalse(createGoldSample() == createGoldSample());
@@ -38,7 +38,7 @@ public class POSSampleTest {
assertFalse(createPredSample().equals(createGoldSample()));
assertFalse(createPredSample().equals(new Object()));
}
-
+
public static POSSample createGoldSample() throws InvalidFormatException {
String sentence = "the_DT stories_NNS about_IN well-heeled_JJ "
+ "communities_NNS and_CC developers_NNS";
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java Fri May 2 12:34:23 2014
@@ -83,7 +83,7 @@ public class POSTaggerFactoryTest {
assertTrue(factory.getSequenceValidator() instanceof DummyPOSSequenceValidator);
assertTrue(factory.getDictionary() instanceof Dictionary);
}
-
+
@Test
public void testPOSTaggerWithDefaultFactory() throws IOException {
POSDictionary posDict = POSDictionary.create(POSDictionaryTest.class
@@ -111,7 +111,7 @@ public class POSTaggerFactoryTest {
assertTrue(factory.getSequenceValidator() instanceof DefaultPOSSequenceValidator);
assertTrue(factory.getDictionary() instanceof Dictionary);
}
-
+
@Test(expected = InvalidFormatException.class)
public void testCreateWithInvalidName() throws InvalidFormatException {
BaseToolFactory.create("X", null);