You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/02/28 18:05:44 UTC
svn commit: r1573000 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind:
NameFinderEventStream.java NameFinderME.java
Author: joern
Date: Fri Feb 28 17:05:44 2014
New Revision: 1573000
URL: http://svn.apache.org/r1573000
Log:
OPENNLP-658 Modified the Name Finder to use the new Sequence Codec support
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java?rev=1573000&r1=1572999&r2=1573000&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java Fri Feb 28 17:05:44 2014
@@ -24,6 +24,7 @@ import java.util.Map;
import opennlp.tools.ml.model.Event;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.SequenceCodec;
import opennlp.tools.util.Span;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
@@ -39,6 +40,8 @@ public class NameFinderEventStream exten
private AdditionalContextFeatureGenerator additionalContextFeatureGenerator = new AdditionalContextFeatureGenerator();
private String type;
+
+ private SequenceCodec<String> codec;
/**
* Creates a new name finder event stream using the specified data stream and context generator.
@@ -46,9 +49,15 @@ public class NameFinderEventStream exten
* @param type null or overrides the type parameter in the provided samples
* @param contextGenerator The context generator used to generate features for the event stream.
*/
- public NameFinderEventStream(ObjectStream<NameSample> dataStream, String type, NameContextGenerator contextGenerator) {
+ public NameFinderEventStream(ObjectStream<NameSample> dataStream, String type, NameContextGenerator contextGenerator, SequenceCodec codec) {
super(dataStream);
+ this.codec = codec;
+
+ if (codec == null) {
+ this.codec = new BioCodec();
+ }
+
this.contextGenerator = contextGenerator;
this.contextGenerator.addFeatureGenerator(new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));
@@ -59,7 +68,7 @@ public class NameFinderEventStream exten
}
public NameFinderEventStream(ObjectStream<NameSample> dataStream) {
- this(dataStream, null, new DefaultNameContextGenerator());
+ this(dataStream, null, new DefaultNameContextGenerator(), null);
}
/**
@@ -113,7 +122,8 @@ public class NameFinderEventStream exten
contextGenerator.clearAdaptiveData();
}
- String outcomes[] = generateOutcomes(sample.getNames(), type, sample.getSentence().length);
+ String outcomes[] = codec.encode(sample.getNames(), sample.getSentence().length);
+// String outcomes[] = generateOutcomes(sample.getNames(), type, sample.getSentence().length);
additionalContextFeatureGenerator.setCurrentContext(sample.getAdditionalContext());
String[] tokens = new String[sample.getSentence().length];
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java?rev=1573000&r1=1572999&r2=1573000&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java Fri Feb 28 17:05:44 2014
@@ -39,6 +39,7 @@ import opennlp.tools.ml.model.MaxentMode
import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Sequence;
+import opennlp.tools.util.SequenceCodec;
import opennlp.tools.util.SequenceValidator;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
@@ -69,6 +70,8 @@ public class NameFinderME implements Tok
public static final String CONTINUE = "cont";
public static final String OTHER = "other";
+ private static SequenceCodec<String> seqCodec = new BilouCodec();
+
protected SequenceClassificationModel<String> model;
protected NameContextGenerator contextGenerator;
@@ -129,6 +132,9 @@ public class NameFinderME implements Tok
if (this.sequenceValidator == null)
this.sequenceValidator = new NameFinderSequenceValidator();
+ // TODO: Remove this!
+ this.sequenceValidator = seqCodec.createSequenceValidator();
+
// if (this.model != null) {
// beam = new BeamSearch<String>(beamSize, contextGenerator, this.model,
// sequenceValidator, beamSize);
@@ -202,37 +208,7 @@ public class NameFinderME implements Tok
contextGenerator.updateAdaptiveData(tokens, c.toArray(new String[c.size()]));
- int start = -1;
- int end = -1;
- List<Span> spans = new ArrayList<Span>(tokens.length);
- for (int li = 0; li < c.size(); li++) {
- String chunkTag = c.get(li);
- if (chunkTag.endsWith(NameFinderME.START)) {
- if (start != -1) {
- spans.add(new Span(start, end, extractNameType(c.get(li - 1))));
- }
-
- start = li;
- end = li + 1;
-
- }
- else if (chunkTag.endsWith(NameFinderME.CONTINUE)) {
- end = li + 1;
- }
- else if (chunkTag.endsWith(NameFinderME.OTHER)) {
- if (start != -1) {
- spans.add(new Span(start, end, extractNameType(c.get(li - 1))));
- start = -1;
- end = -1;
- }
- }
- }
-
- if (start != -1) {
- spans.add(new Span(start, end, extractNameType(c.get(c.size() - 1))));
- }
-
- return spans.toArray(new Span[spans.size()]);
+ return seqCodec.decode(c);
}
/**
@@ -322,6 +298,8 @@ public class NameFinderME implements Tok
public static TokenNameFinderModel train(String languageCode, String type, ObjectStream<NameSample> samples,
TrainingParameters trainParams, AdaptiveFeatureGenerator generator, final Map<String, Object> resources) throws IOException {
+ // SequenceCodec seqCodec = new BiolouCodec();
+
if (languageCode == null) {
throw new IllegalArgumentException("languageCode must not be null!");
}
@@ -343,7 +321,7 @@ public class NameFinderME implements Tok
if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) {
ObjectStream<Event> eventStream = new NameFinderEventStream(samples, type,
- new DefaultNameContextGenerator(featureGenerator));
+ new DefaultNameContextGenerator(featureGenerator), seqCodec);
EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams.getSettings(), manifestInfoEntries);
nameFinderModel = trainer.train(eventStream);