You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2017/06/06 15:20:18 UTC
opennlp git commit: OPENNLP-1086: Refactor the Data Indexers
Repository: opennlp
Updated Branches:
refs/heads/master a13691b5b -> 1484f1d63
OPENNLP-1086: Refactor the Data Indexers
The following has been done:
- Use Java 8 streams where it makes sense
- Deduplicate the index method and have one common one
- Avoid having all predicate Strings twice in memory for cutoff filter
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/1484f1d6
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/1484f1d6
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/1484f1d6
Branch: refs/heads/master
Commit: 1484f1d637b4d33490d15a905d8d736de850e9b5
Parents: a13691b
Author: Jörn Kottmann <jo...@apache.org>
Authored: Tue May 30 11:21:14 2017 +0200
Committer: Jörn Kottmann <jo...@apache.org>
Committed: Tue Jun 6 17:10:25 2017 +0200
----------------------------------------------------------------------
.../tools/ml/model/AbstractDataIndexer.java | 67 +++++++++---
.../tools/ml/model/OnePassDataIndexer.java | 95 ++++-------------
.../ml/model/OnePassRealValueDataIndexer.java | 55 +---------
.../tools/ml/model/TwoPassDataIndexer.java | 101 +++++--------------
.../opennlp/tools/eval/ArvoresDeitadasEval.java | 4 +-
.../tools/eval/Conll02NameFinderEval.java | 12 +--
.../opennlp/tools/eval/ConllXPosTaggerEval.java | 2 +-
7 files changed, 110 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
index be6a6e4..3a3fb73 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
@@ -17,14 +17,20 @@
package opennlp.tools.ml.model;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import opennlp.tools.ml.AbstractTrainer;
import opennlp.tools.util.InsufficientTrainingDataException;
+import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
@@ -152,6 +158,35 @@ public abstract class AbstractDataIndexer implements DataIndexer {
return numUniqueEvents;
}
+ protected List<ComparableEvent> index(ObjectStream<Event> events,
+ Map<String, Integer> predicateIndex) throws IOException {
+ Map<String, Integer> omap = new HashMap<>();
+
+ List<ComparableEvent> eventsToCompare = new ArrayList<>();
+
+ Event ev;
+ while ((ev = events.read()) != null) {
+
+ omap.putIfAbsent(ev.getOutcome(), omap.size());
+
+ int[] cons = Arrays.stream(ev.getContext())
+ .map(pred -> predicateIndex.get(pred))
+ .filter(Objects::nonNull)
+ .mapToInt(i -> i).toArray();
+
+ // drop events with no active features
+ if (cons.length > 0) {
+ int ocID = omap.get(ev.getOutcome());
+ eventsToCompare.add(new ComparableEvent(ocID, cons, ev.getValues()));
+ } else {
+ display("Dropped event " + ev.getOutcome() + ":"
+ + Arrays.asList(ev.getContext()) + "\n");
+ }
+ }
+ outcomeLabels = toIndexedStringArray(omap);
+ predLabels = toIndexedStringArray(predicateIndex);
+ return eventsToCompare;
+ }
public int getNumEvents() {
return numEvents;
@@ -163,17 +198,14 @@ public abstract class AbstractDataIndexer implements DataIndexer {
* @param predicateSet The set of predicates which will be used for model building.
* @param counter The predicate counters.
* @param cutoff The cutoff which determines whether a predicate is included.
+ * @deprecated will be removed after 1.8.1 release
*/
+ @Deprecated
protected static void update(String[] ec, Set<String> predicateSet,
Map<String,Integer> counter, int cutoff) {
for (String s : ec) {
- Integer i = counter.get(s);
- if (i == null) {
- counter.put(s, 1);
- }
- else {
- counter.put(s, i + 1);
- }
+ counter.merge(s, 1, (value, one) -> value + one);
+
if (!predicateSet.contains(s) && counter.get(s) >= cutoff) {
predicateSet.add(s);
}
@@ -181,6 +213,17 @@ public abstract class AbstractDataIndexer implements DataIndexer {
}
/**
+ * Updates the set of predicated and counter with the specified event contexts.
+ * @param ec The contexts/features which occur in a event.
+ * @param counter The predicate counters.
+ */
+ protected static void update(String[] ec, Map<String,Integer> counter) {
+ for (String s : ec) {
+ counter.merge(s, 1, (value, one) -> value + one);
+ }
+ }
+
+ /**
* Utility method for creating a String[] array from a map whose
* keys are labels (Strings) to be stored in the array and whose
* values are the indices (Integers) at which the corresponding
@@ -188,14 +231,10 @@ public abstract class AbstractDataIndexer implements DataIndexer {
*
* @param labelToIndexMap a <code>TObjectIntHashMap</code> value
* @return a <code>String[]</code> value
- * @since maxent 1.2.6
*/
- protected static String[] toIndexedStringArray(Map<String,Integer> labelToIndexMap) {
- final String[] array = new String[labelToIndexMap.size()];
- for (String label : labelToIndexMap.keySet()) {
- array[labelToIndexMap.get(label)] = label;
- }
- return array;
+ protected static String[] toIndexedStringArray(Map<String, Integer> labelToIndexMap) {
+ return labelToIndexMap.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue))
+ .map(Map.Entry::getKey).toArray(String[]::new);
}
public float[][] getValues() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
index cf8fb7f..2f9c719 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
@@ -18,17 +18,13 @@
package opennlp.tools.ml.model;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ObjectStreamUtils;
/**
* An indexer for maxent model data which handles cutoffs for uncommon
@@ -44,28 +40,24 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT);
boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT);
- Map<String, Integer> predicateIndex = new HashMap<>();
- List<Event> events;
- List<ComparableEvent> eventsToCompare;
+ long start = System.currentTimeMillis();
- display("Indexing events using cutoff of " + cutoff + "\n\n");
+ display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n");
display("\tComputing event counts... ");
- events = computeEventCounts(eventStream, predicateIndex, cutoff);
+ Map<String, Integer> predicateIndex = new HashMap<>();
+ List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff);
display("done. " + events.size() + " events\n");
display("\tIndexing... ");
- eventsToCompare = index(events, predicateIndex);
- // done with event list
- events = null;
- // done with predicates
- predicateIndex = null;
+ List<ComparableEvent> eventsToCompare =
+ index(ObjectStreamUtils.createObjectStream(events), predicateIndex);
display("done.\n");
display("Sorting and merging events... ");
sortAndMerge(eventsToCompare, sort);
- display("Done indexing.\n");
+ display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d));
}
/**
@@ -84,71 +76,26 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
*/
private List<Event> computeEventCounts(ObjectStream<Event> eventStream,
Map<String, Integer> predicatesInOut, int cutoff) throws IOException {
- Set<String> predicateSet = new HashSet<>();
+
Map<String, Integer> counter = new HashMap<>();
List<Event> events = new LinkedList<>();
Event ev;
while ((ev = eventStream.read()) != null) {
events.add(ev);
- update(ev.getContext(), predicateSet, counter, cutoff);
+ update(ev.getContext(), counter);
}
- predCounts = new int[predicateSet.size()];
- int index = 0;
- for (Iterator<String> pi = predicateSet.iterator(); pi.hasNext(); index++) {
- String predicate = pi.next();
- predCounts[index] = counter.get(predicate);
- predicatesInOut.put(predicate, index);
- }
- return events;
- }
- protected List<ComparableEvent> index(List<Event> events,
- Map<String, Integer> predicateIndex) {
- Map<String, Integer> omap = new HashMap<>();
-
- int numEvents = events.size();
- int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
- List<Integer> indexedContext = new ArrayList<>();
-
- for (Event ev:events) {
- String[] econtext = ev.getContext();
- ComparableEvent ce;
-
- int ocID;
- String oc = ev.getOutcome();
-
- if (omap.containsKey(oc)) {
- ocID = omap.get(oc);
- } else {
- ocID = outcomeCount++;
- omap.put(oc, ocID);
- }
-
- for (String pred : econtext) {
- if (predicateIndex.containsKey(pred)) {
- indexedContext.add(predicateIndex.get(pred));
- }
- }
-
- // drop events with no active features
- if (indexedContext.size() > 0) {
- int[] cons = new int[indexedContext.size()];
- for (int ci = 0; ci < cons.length; ci++) {
- cons[ci] = indexedContext.get(ci);
- }
- ce = new ComparableEvent(ocID, cons);
- eventsToCompare.add(ce);
- } else {
- display("Dropped event " + ev.getOutcome() + ":"
- + Arrays.asList(ev.getContext()) + "\n");
- }
- // recycle the TIntArrayList
- indexedContext.clear();
+ String[] predicateSet = counter.entrySet().stream()
+ .filter(entry -> entry.getValue() >= cutoff)
+ .map(Map.Entry::getKey).sorted()
+ .toArray(String[]::new);
+
+ predCounts = new int[predicateSet.length];
+ for (int i = 0; i < predicateSet.length; i++) {
+ predCounts[i] = counter.get(predicateSet[i]);
+ predicatesInOut.put(predicateSet[i], i);
}
- outcomeLabels = toIndexedStringArray(omap);
- predLabels = toIndexedStringArray(predicateIndex);
- return eventsToCompare;
- }
+ return events;
+ }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
index a5abb34..2cb23b2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
@@ -17,11 +17,7 @@
package opennlp.tools.ml.model;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import opennlp.tools.util.InsufficientTrainingDataException;
@@ -41,7 +37,7 @@ public class OnePassRealValueDataIndexer extends OnePassDataIndexer {
return values;
}
- protected int sortAndMerge(List<ComparableEvent> eventsToCompare,boolean sort)
+ protected int sortAndMerge(List<ComparableEvent> eventsToCompare, boolean sort)
throws InsufficientTrainingDataException {
int numUniqueEvents = super.sortAndMerge(eventsToCompare,sort);
values = new float[numUniqueEvents][];
@@ -55,53 +51,4 @@ public class OnePassRealValueDataIndexer extends OnePassDataIndexer {
}
return numUniqueEvents;
}
-
- @Override
- protected List<ComparableEvent> index(List<Event> events, Map<String,Integer> predicateIndex) {
- Map<String,Integer> omap = new HashMap<>();
-
- int numEvents = events.size();
- int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
- List<Integer> indexedContext = new ArrayList<>();
-
- for (Event ev:events) {
- String[] econtext = ev.getContext();
- ComparableEvent ce;
-
- int ocID;
- String oc = ev.getOutcome();
-
- if (omap.containsKey(oc)) {
- ocID = omap.get(oc);
- } else {
- ocID = outcomeCount++;
- omap.put(oc, ocID);
- }
-
- for (String pred : econtext) {
- if (predicateIndex.containsKey(pred)) {
- indexedContext.add(predicateIndex.get(pred));
- }
- }
-
- //drop events with no active features
- if (indexedContext.size() > 0) {
- int[] cons = new int[indexedContext.size()];
- for (int ci = 0; ci < cons.length; ci++) {
- cons[ci] = indexedContext.get(ci);
- }
- ce = new ComparableEvent(ocID, cons, ev.getValues());
- eventsToCompare.add(ce);
- } else {
- System.err.println("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()));
- }
-
- indexedContext.clear();
- }
- outcomeLabels = toIndexedStringArray(omap);
- predLabels = toIndexedStringArray(predicateIndex);
- return eventsToCompare;
- }
-
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
index b3cc89b..5e347e8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
@@ -23,14 +23,10 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import opennlp.tools.util.ObjectStream;
@@ -52,26 +48,30 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT);
boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT);
- Map<String,Integer> predicateIndex = new HashMap<>();
- List<ComparableEvent> eventsToCompare;
+ long start = System.currentTimeMillis();
- display("Indexing events using cutoff of " + cutoff + "\n\n");
+ display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n");
display("\tComputing event counts... ");
+ Map<String,Integer> predicateIndex = new HashMap<>();
+
File tmp = File.createTempFile("events", null);
tmp.deleteOnExit();
- Writer osw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmp),"UTF8"));
- int numEvents = computeEventCounts(eventStream, osw, predicateIndex, cutoff);
+ int numEvents;
+ try (Writer osw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmp),
+ StandardCharsets.UTF_8))) {
+ numEvents = computeEventCounts(eventStream, osw, predicateIndex, cutoff);
+ }
display("done. " + numEvents + " events\n");
display("\tIndexing... ");
+ List<ComparableEvent> eventsToCompare;
try (FileEventStream fes = new FileEventStream(tmp)) {
- eventsToCompare = index(numEvents, fes, predicateIndex);
+ eventsToCompare = index(fes, predicateIndex);
}
- // done with predicates
- predicateIndex = null;
+
tmp.delete();
display("done.\n");
@@ -82,9 +82,9 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
display("Collecting events... ");
}
sortAndMerge(eventsToCompare,sort);
- display("Done indexing.\n");
-
+ display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d));
}
+
/**
* Reads events from <tt>eventStream</tt> into a linked list. The
* predicates associated with each event are counted and any which
@@ -100,75 +100,26 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
Map<String,Integer> predicatesInOut, int cutoff) throws IOException {
Map<String,Integer> counter = new HashMap<>();
int eventCount = 0;
- Set<String> predicateSet = new HashSet<>();
Event ev;
while ((ev = eventStream.read()) != null) {
eventCount++;
eventStore.write(FileEventStream.toLine(ev));
String[] ec = ev.getContext();
- update(ec,predicateSet,counter,cutoff);
+ update(ec, counter);
}
- predCounts = new int[predicateSet.size()];
- int index = 0;
- for (Iterator<String> pi = predicateSet.iterator(); pi.hasNext(); index++) {
- String predicate = pi.next();
- predCounts[index] = counter.get(predicate);
- predicatesInOut.put(predicate,index);
- }
- eventStore.close();
- return eventCount;
- }
-
- // TODO: merge this code with the copy and paste version in OnePassDataIndexer
- private List<ComparableEvent> index(int numEvents, ObjectStream<Event> es,
- Map<String,Integer> predicateIndex) throws IOException {
- Map<String,Integer> omap = new HashMap<>();
- int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
- List<Integer> indexedContext = new ArrayList<>();
+ String[] predicateSet = counter.entrySet().stream()
+ .filter(entry -> entry.getValue() >= cutoff)
+ .map(Map.Entry::getKey).sorted()
+ .toArray(String[]::new);
- Event ev;
- while ((ev = es.read()) != null) {
- String[] econtext = ev.getContext();
- ComparableEvent ce;
-
- int ocID;
- String oc = ev.getOutcome();
-
- if (omap.containsKey(oc)) {
- ocID = omap.get(oc);
- }
- else {
- ocID = outcomeCount++;
- omap.put(oc, ocID);
- }
-
- for (String pred : econtext) {
- if (predicateIndex.containsKey(pred)) {
- indexedContext.add(predicateIndex.get(pred));
- }
- }
-
- // drop events with no active features
- if (indexedContext.size() > 0) {
- int[] cons = new int[indexedContext.size()];
- for (int ci = 0;ci < cons.length; ci++) {
- cons[ci] = indexedContext.get(ci);
- }
- ce = new ComparableEvent(ocID, cons);
- eventsToCompare.add(ce);
- }
- else {
- display("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()) + "\n");
- }
- // recycle the TIntArrayList
- indexedContext.clear();
+ predCounts = new int[predicateSet.length];
+ for (int i = 0; i < predicateSet.length; i++) {
+ predCounts[i] = counter.get(predicateSet[i]);
+ predicatesInOut.put(predicateSet[i], i);
}
- outcomeLabels = toIndexedStringArray(omap);
- predLabels = toIndexedStringArray(predicateIndex);
- return eventsToCompare;
+
+ return eventCount;
}
}
-
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
index 7e55165..dd54480 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
@@ -199,7 +199,7 @@ public class ArvoresDeitadasEval {
@Test
public void evalPortugueseChunkerQn() throws IOException {
chunkerCrossEval(EvalUtil.createMaxentQnParams(),
- 0.9652111035230788d);
+ 0.9648211936491359d);
}
@Test
@@ -208,7 +208,7 @@ public class ArvoresDeitadasEval {
params.put("Threads", 4);
// NOTE: Should be the same as without multiple threads!!!
- chunkerCrossEval(params, 0.9647304571382662);
+ chunkerCrossEval(params, 0.9649180953528779d);
}
@Test
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/test/java/opennlp/tools/eval/Conll02NameFinderEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll02NameFinderEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll02NameFinderEval.java
index abe53aa..c064e3f 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll02NameFinderEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll02NameFinderEval.java
@@ -346,10 +346,10 @@ public class Conll02NameFinderEval {
Conll02NameSampleStream.GENERATE_PERSON_ENTITIES);
eval(maxentModel, spanishTestAFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_PERSON_ENTITIES, 0.7432498772704957d);
+ Conll02NameSampleStream.GENERATE_PERSON_ENTITIES, 0.7454634624816087d);
eval(maxentModel, spanishTestBFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_PERSON_ENTITIES, 0.8218773096821878d);
+ Conll02NameSampleStream.GENERATE_PERSON_ENTITIES, 0.8215339233038348d);
}
@Test
@@ -388,10 +388,10 @@ public class Conll02NameFinderEval {
Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES);
eval(maxentModel, spanishTestAFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 0.6827859978347167d);
+ Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 0.682961897915169d);
eval(maxentModel, spanishTestBFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 0.7766212970376302d);
+ Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 0.7776447105788423d);
}
@Test
@@ -472,10 +472,10 @@ public class Conll02NameFinderEval {
Conll02NameSampleStream.GENERATE_MISC_ENTITIES);
eval(maxentModel, spanishTestAFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_MISC_ENTITIES, 0.47095761381475676d);
+ Conll02NameSampleStream.GENERATE_MISC_ENTITIES, 0.470219435736677d);
eval(maxentModel, spanishTestBFile, LANGUAGE.ES,
- Conll02NameSampleStream.GENERATE_MISC_ENTITIES, 0.4926931106471817d);
+ Conll02NameSampleStream.GENERATE_MISC_ENTITIES, 0.5020576131687243d);
}
@Test
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1484f1d6/opennlp-tools/src/test/java/opennlp/tools/eval/ConllXPosTaggerEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/ConllXPosTaggerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/ConllXPosTaggerEval.java
index 8ac7b3a..98a0ded 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/ConllXPosTaggerEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/ConllXPosTaggerEval.java
@@ -163,6 +163,6 @@ public class ConllXPosTaggerEval {
"conllx/data/swedish/talbanken05/train/swedish_talbanken05_train.conll"), "se", params);
eval(maxentModel, new File(EvalUtil.getOpennlpDataDir(),
- "conllx/data/swedish/talbanken05/test/swedish_talbanken05_test.conll"), 0.9322842998585573d);
+ "conllx/data/swedish/talbanken05/test/swedish_talbanken05_test.conll"), 0.9347595473833098d);
}
}