You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/01/30 14:27:54 UTC
svn commit: r1562813 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind:
DefaultNameContextGenerator.java NameSampleSequenceStream.java
Author: joern
Date: Thu Jan 30 13:27:54 2014
New Revision: 1562813
URL: http://svn.apache.org/r1562813
Log:
OPENNLP-641 Passing in the outcomes is now optinal
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java?rev=1562813&r1=1562812&r2=1562813&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java Thu Jan 30 13:27:54 2014
@@ -122,18 +122,21 @@ public class DefaultNameContextGenerator
String po = NameFinderME.OTHER;
String ppo = NameFinderME.OTHER;
- if (index > 1){
- ppo = preds[index-2];
+ // TODO: These should be moved out here in its own feature generator!
+ if (preds != null) {
+ if (index > 1){
+ ppo = preds[index-2];
+ }
+
+ if (index > 0) {
+ po = preds[index-1];
+ }
+ features.add("po=" + po);
+ features.add("pow=" + po + "," + tokens[index]);
+ features.add("powf=" + po + "," + FeatureGeneratorUtil.tokenFeature(tokens[index]));
+ features.add("ppo=" + ppo);
}
-
- if (index > 0) {
- po = preds[index-1];
- }
- features.add("po=" + po);
- features.add("pow=" + po + "," + tokens[index]);
- features.add("powf=" + po + "," + FeatureGeneratorUtil.tokenFeature(tokens[index]));
- features.add("ppo=" + ppo);
-
+
return features.toArray(new String[features.size()]);
}
}
\ No newline at end of file
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java?rev=1562813&r1=1562812&r2=1562813&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java Thu Jan 30 13:27:54 2014
@@ -34,18 +34,30 @@ public class NameSampleSequenceStream im
private NameContextGenerator pcg;
private List<NameSample> samples;
+ private final boolean useOutcomes;
public NameSampleSequenceStream(ObjectStream<NameSample> psi) throws IOException {
- this(psi, new DefaultNameContextGenerator((AdaptiveFeatureGenerator) null));
+ this(psi, new DefaultNameContextGenerator((AdaptiveFeatureGenerator) null), true);
}
public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen)
throws IOException {
- this(psi, new DefaultNameContextGenerator(featureGen));
+ this(psi, new DefaultNameContextGenerator(featureGen), true);
+ }
+
+ public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen, boolean useOutcomes)
+ throws IOException {
+ this(psi, new DefaultNameContextGenerator(featureGen), useOutcomes);
}
public NameSampleSequenceStream(ObjectStream<NameSample> psi, NameContextGenerator pcg)
throws IOException {
+ this(psi, pcg, true);
+ }
+
+ public NameSampleSequenceStream(ObjectStream<NameSample> psi, NameContextGenerator pcg, boolean useOutcomes)
+ throws IOException {
+ this.useOutcomes = useOutcomes;
samples = new ArrayList<NameSample>();
NameSample sample;
@@ -74,7 +86,7 @@ public class NameSampleSequenceStream im
@SuppressWarnings("unchecked")
public Iterator<Sequence> iterator() {
- return new NameSampleSequenceIterator(samples.iterator());
+ return new NameSampleSequenceIterator(samples.iterator(), useOutcomes);
}
}
@@ -83,9 +95,11 @@ class NameSampleSequenceIterator impleme
private Iterator<NameSample> psi;
private NameContextGenerator cg;
+ private boolean useOutcomes;
- public NameSampleSequenceIterator(Iterator<NameSample> psi) {
+ public NameSampleSequenceIterator(Iterator<NameSample> psi, boolean useOutcomes) {
this.psi = psi;
+ this.useOutcomes = useOutcomes;
cg = new DefaultNameContextGenerator(null);
}
@@ -104,8 +118,14 @@ class NameSampleSequenceIterator impleme
// it is safe to pass the tags as previous tags because
// the context generator does not look for non predicted tags
- String[] context = cg.getContext(i, sentence, tags, null);
-
+ String[] context;
+ if (useOutcomes) {
+ context = cg.getContext(i, sentence, tags, null);
+ }
+ else {
+ context = cg.getContext(i, sentence, null, null);
+ }
+
events[i] = new Event(tags[i], context);
}
Sequence<NameSample> sequence = new Sequence<NameSample>(events,sample);