You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/02/20 15:53:17 UTC
svn commit: r1570210 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools:
chunker/ChunkSampleSequenceStream.java chunker/ChunkerME.java
chunker/ChunkerModel.java ml/BeamSearch.java
ml/model/SequenceClassificationModel.java
Author: joern
Date: Thu Feb 20 14:53:16 2014
New Revision: 1570210
URL: http://svn.apache.org/r1570210
Log:
OPENNLP-641 Added initial sequence classification support
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java
Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java?rev=1570210&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java (added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java Thu Feb 20 14:53:16 2014
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.chunker;
+
+import java.io.IOException;
+
+import opennlp.tools.ml.model.AbstractModel;
+import opennlp.tools.ml.model.Event;
+import opennlp.tools.ml.model.Sequence;
+import opennlp.tools.ml.model.SequenceStream;
+import opennlp.tools.util.ObjectStream;
+
+public class ChunkSampleSequenceStream implements SequenceStream {
+
+ private final ObjectStream<ChunkSample> samples;
+ private final ChunkerContextGenerator contextGenerator;
+
+ public ChunkSampleSequenceStream(ObjectStream<ChunkSample> samples,
+ ChunkerContextGenerator contextGenerator) {
+ this.samples = samples;
+ this.contextGenerator = contextGenerator;
+ }
+
+ @Override
+ public Sequence read() throws IOException {
+ ChunkSample sample = samples.read();
+
+ if (sample != null) {
+ String sentence[] = sample.getSentence();
+ String tags[] = sample.getTags();
+ Event[] events = new Event[sentence.length];
+
+ for (int i=0; i < sentence.length; i++) {
+
+ // it is safe to pass the tags as previous tags because
+ // the context generator does not look for non predicted tags
+ String[] context = contextGenerator.getContext(i, sentence, tags, null);
+
+ events[i] = new Event(tags[i], context);
+ }
+ return new Sequence<ChunkSample>(events,sample);
+ }
+
+ return null;
+ }
+
+ @Override
+ public Event[] updateContext(Sequence sequence, AbstractModel model) {
+ // TODO: Should be implemented for Perceptron sequence learning ...
+ return null;
+ }
+
+ @Override
+ public void reset() throws IOException, UnsupportedOperationException {
+ samples.reset();
+ }
+
+ @Override
+ public void close() throws IOException {
+ samples.close();
+ }
+
+}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java Thu Feb 20 14:53:16 2014
@@ -22,10 +22,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import opennlp.tools.ml.EventTrainer;
+import opennlp.tools.ml.SequenceTrainer;
+import opennlp.tools.ml.TrainerFactory;
+import opennlp.tools.ml.TrainerFactory.TrainerType;
import opennlp.tools.ml.model.Event;
import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.ml.model.TrainUtil;
-import opennlp.tools.util.BeamSearch;
+import opennlp.tools.postag.POSSampleSequenceStream;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Sequence;
import opennlp.tools.util.SequenceValidator;
@@ -40,17 +45,15 @@ public class ChunkerME implements Chunke
public static final int DEFAULT_BEAM_SIZE = 10;
- /**
- * The beam used to search for sequences of chunk tag assignments.
- */
- protected BeamSearch<String> beam;
-
private Sequence bestSequence;
/**
* The model used to assign chunk tags to a sequence of tokens.
*/
- protected MaxentModel model;
+ protected SequenceClassificationModel<String> model;
+
+ private ChunkerContextGenerator contextGenerator;
+ private SequenceValidator<String> sequenceValidator;
/**
* Initializes the current instance with the specified model and
@@ -64,10 +67,20 @@ public class ChunkerME implements Chunke
* @deprecated Use {@link #ChunkerME(ChunkerModel, int)} instead
* and use the {@link ChunkerFactory} to configure the {@link SequenceValidator} and {@link ChunkerContextGenerator}.
*/
+ @Deprecated
public ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<String> sequenceValidator,
ChunkerContextGenerator contextGenerator) {
- this.model = model.getChunkerModel();
- beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator, 0);
+
+ this.sequenceValidator = sequenceValidator;
+ this.contextGenerator = contextGenerator;
+
+ if (model.getChunkerModel() != null) {
+ this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ model.getChunkerModel(), 0);
+ }
+ else {
+ this.model = model.getChunkerSequenceModel();
+ }
}
/**
@@ -82,12 +95,13 @@ public class ChunkerME implements Chunke
* @deprecated Use {@link #ChunkerME(ChunkerModel, int)} instead
* and use the {@link ChunkerFactory} to configure the {@link SequenceValidator}.
*/
+ @Deprecated
public ChunkerME(ChunkerModel model, int beamSize,
SequenceValidator<String> sequenceValidator) {
this(model, beamSize, sequenceValidator,
new DefaultChunkerContextGenerator());
}
-
+
/**
* Initializes the current instance with the specified model and
* the specified beam size.
@@ -96,10 +110,18 @@ public class ChunkerME implements Chunke
* @param beamSize The size of the beam that should be used when decoding sequences.
*/
public ChunkerME(ChunkerModel model, int beamSize) {
- this.model = model.getChunkerModel();
- ChunkerContextGenerator contextGenerator = model.getFactory().getContextGenerator();
- SequenceValidator<String> sequenceValidator = model.getFactory().getSequenceValidator();
- beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator, 0);
+
+ contextGenerator = model.getFactory().getContextGenerator();
+ sequenceValidator = model.getFactory().getSequenceValidator();
+ // beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator, 0);
+
+ if (model.getChunkerModel() != null) {
+ this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ model.getChunkerModel(), 0);
+ }
+ else {
+ this.model = model.getChunkerSequenceModel();
+ }
}
/**
@@ -115,12 +137,13 @@ public class ChunkerME implements Chunke
@Deprecated
public List<String> chunk(List<String> toks, List<String> tags) {
bestSequence =
- beam.bestSequence(toks.toArray(new String[toks.size()]), new Object[] { tags.toArray(new String[tags.size()]) });
+ model.bestSequence(toks.toArray(new String[toks.size()]), new Object[] { tags.toArray(new String[tags.size()]) },
+ contextGenerator, sequenceValidator);
return bestSequence.getOutcomes();
}
public String[] chunk(String[] toks, String[] tags) {
- bestSequence = beam.bestSequence(toks, new Object[] {tags});
+ bestSequence = model.bestSequence(toks, new Object[] {tags}, contextGenerator, sequenceValidator);
List<String> c = bestSequence.getOutcomes();
return c.toArray(new String[c.size()]);
}
@@ -137,12 +160,13 @@ public class ChunkerME implements Chunke
}
public Sequence[] topKSequences(String[] sentence, String[] tags) {
- return beam.bestSequences(DEFAULT_BEAM_SIZE, sentence,
- new Object[] { tags });
+ return model.bestSequences(DEFAULT_BEAM_SIZE, sentence,
+ new Object[] { tags }, contextGenerator, sequenceValidator);
}
public Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore) {
- return beam.bestSequences(DEFAULT_BEAM_SIZE, sentence, new Object[] { tags },minSequenceScore);
+ return model.bestSequences(DEFAULT_BEAM_SIZE, sentence, new Object[] { tags }, minSequenceScore,
+ contextGenerator, sequenceValidator);
}
/**
@@ -171,12 +195,37 @@ public class ChunkerME implements Chunke
Map<String, String> manifestInfoEntries = new HashMap<String, String>();
- ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator());
-
- MaxentModel maxentModel = TrainUtil.train(es, mlParams.getSettings(),
- manifestInfoEntries);
+ TrainerType trainerType = TrainerFactory.getTrainerType(mlParams.getSettings());
+
- return new ChunkerModel(lang, maxentModel, manifestInfoEntries, factory);
+ MaxentModel chunkerModel = null;
+ SequenceClassificationModel<String> seqChunkerModel = null;
+
+ if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) {
+ ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator());
+ EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams.getSettings(),
+ manifestInfoEntries);
+ chunkerModel = trainer.train(es);
+ }
+ else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) {
+ SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer(
+ mlParams.getSettings(), manifestInfoEntries);
+
+ // TODO: This will probably cause issue, since the feature generator uses the outcomes array
+
+ ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator());
+ seqChunkerModel = trainer.train(ss);
+ }
+ else {
+ throw new IllegalArgumentException("Trainer type is not supported: " + trainerType);
+ }
+
+ if (chunkerModel != null) {
+ return new ChunkerModel(lang, chunkerModel, manifestInfoEntries, factory);
+ }
+ else {
+ return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory);
+ }
}
/**
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java Thu Feb 20 14:53:16 2014
@@ -31,6 +31,7 @@ import opennlp.tools.ml.model.AbstractMo
import opennlp.tools.ml.model.BinaryFileDataReader;
import opennlp.tools.ml.model.GenericModelReader;
import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.BaseToolFactory;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.BaseModel;
@@ -55,6 +56,14 @@ public class ChunkerModel extends BaseMo
this(languageCode, chunkerModel, manifestInfoEntries, new ChunkerFactory());
}
+ public ChunkerModel(String languageCode, SequenceClassificationModel<String> chunkerModel,
+ Map<String, String> manifestInfoEntries, ChunkerFactory factory) {
+ super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
+ artifactMap.put(CHUNKER_MODEL_ENTRY_NAME, chunkerModel);
+ checkArtifactMap();
+ }
+
+
public ChunkerModel(String languageCode, MaxentModel chunkerModel,
Map<String, String> manifestInfoEntries, ChunkerFactory factory) {
super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
@@ -97,7 +106,21 @@ public class ChunkerModel extends BaseMo
}
public MaxentModel getChunkerModel() {
- return (MaxentModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+ if (artifactMap.get(CHUNKER_MODEL_ENTRY_NAME) instanceof MaxentModel) {
+ return (MaxentModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+ }
+ else {
+ return null;
+ }
+ }
+
+ public SequenceClassificationModel<String> getChunkerSequenceModel() {
+ if (artifactMap.get(CHUNKER_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
+ return (SequenceClassificationModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+ }
+ else {
+ return null;
+ }
}
@Override
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java Thu Feb 20 14:53:16 2014
@@ -80,7 +80,7 @@ public class BeamSearch<T> implements Se
* @return The top ranked sequence of outcomes or null if no sequence could be found
*/
public Sequence[] bestSequences(int numSequences, T[] sequence,
- Object[] additionalContext, BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator) {
+ Object[] additionalContext, double minSequenceScore, BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator) {
Heap<Sequence> prev = new ListHeap<Sequence>(size);
Heap<Sequence> next = new ListHeap<Sequence>(size);
@@ -126,9 +126,9 @@ public class BeamSearch<T> implements Se
String out = model.getOutcome(p);
if (validator.validSequence(i, sequence, outcomes, out)) {
Sequence ns = new Sequence(top, out, scores[p]);
- // if (ns.getScore() > minSequenceScore) {
+ if (ns.getScore() > minSequenceScore) {
next.add(ns);
- // }
+ }
}
}
@@ -137,9 +137,9 @@ public class BeamSearch<T> implements Se
String out = model.getOutcome(p);
if (validator.validSequence(i, sequence, outcomes, out)) {
Sequence ns = new Sequence(top, out, scores[p]);
- // if (ns.getScore() > minSequenceScore) {
+ if (ns.getScore() > minSequenceScore) {
next.add(ns);
- //}
+ }
}
}
}
@@ -162,6 +162,11 @@ public class BeamSearch<T> implements Se
return topSequences;
}
+ public Sequence[] bestSequences(int numSequences, T[] sequence,
+ Object[] additionalContext, BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator) {
+ return bestSequences(numSequences, sequence, additionalContext, zeroLog, cg, validator);
+ }
+
public Sequence bestSequence(T[] sequence, Object[] additionalContext,
BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator) {
Sequence sequences[] = bestSequences(1, sequence, additionalContext, cg, validator);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java Thu Feb 20 14:53:16 2014
@@ -42,6 +42,19 @@ public interface SequenceClassificationM
*/
Sequence bestSequence(T[] sequence, Object[] additionalContext,
BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator);
+
+ /**
+ * Finds the n most probable sequences.
+ *
+ * @param sequence
+ * @param additionalContext
+ * @param cg
+ * @param validator
+ *
+ * @return
+ */
+ Sequence[] bestSequences(int numSequences, T[] sequence,
+ Object[] additionalContext, double minSequenceScore, BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator);
/**
* Finds the n most probable sequences.