You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/03/07 11:15:05 UTC

svn commit: r1575219 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: cmdline/namefind/ namefind/

Author: joern
Date: Fri Mar  7 10:15:04 2014
New Revision: 1575219

URL: http://svn.apache.org/r1575219
Log:
OPENNLP-658 The sequence codec is now configurable in the name finder.

Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TrainingParams.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java Fri Mar  7 10:15:04 2014
@@ -28,10 +28,14 @@ import opennlp.tools.cmdline.TerminateTo
 import opennlp.tools.cmdline.namefind.TokenNameFinderCrossValidatorTool.CVToolParams;
 import opennlp.tools.cmdline.params.CVParams;
 import opennlp.tools.cmdline.params.DetailedFMeasureEvaluatorParams;
+import opennlp.tools.namefind.BilouCodec;
+import opennlp.tools.namefind.BioCodec;
 import opennlp.tools.namefind.NameSample;
 import opennlp.tools.namefind.NameSampleTypeFilter;
 import opennlp.tools.namefind.TokenNameFinderCrossValidator;
 import opennlp.tools.namefind.TokenNameFinderEvaluationMonitor;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.SequenceCodec;
 import opennlp.tools.util.eval.EvaluationMonitor;
 import opennlp.tools.util.model.ModelUtil;
 
@@ -78,10 +82,21 @@ public final class TokenNameFinderCrossV
       listeners.add(detailedFListener);
     }
 
+    String sequenceCodecImplName = params.getSequenceCodec();
+    
+    if ("BIO".equals(sequenceCodecImplName)) {
+      sequenceCodecImplName = BioCodec.class.getName();
+    }
+    else if ("BILOU".equals(sequenceCodecImplName)) {
+      sequenceCodecImplName = BilouCodec.class.getName();
+    }
+    
+    SequenceCodec<String> sequenceCodec = TokenNameFinderModel.instantiateSequenceCodec(sequenceCodecImplName);
+    
     TokenNameFinderCrossValidator validator;
     try {
       validator = new TokenNameFinderCrossValidator(params.getLang(),
-          params.getType(), mlParams, featureGeneratorBytes, resources,
+          params.getType(), mlParams, featureGeneratorBytes, resources, sequenceCodec,
           listeners.toArray(new TokenNameFinderEvaluationMonitor[listeners.size()]));
       validator.evaluate(sampleStream, params.getFolds());
     } catch (IOException e) {

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java Fri Mar  7 10:15:04 2014
@@ -28,10 +28,13 @@ import opennlp.tools.cmdline.CmdLineUtil
 import opennlp.tools.cmdline.TerminateToolException;
 import opennlp.tools.cmdline.namefind.TokenNameFinderTrainerTool.TrainerToolParams;
 import opennlp.tools.cmdline.params.TrainingToolParams;
+import opennlp.tools.namefind.BilouCodec;
+import opennlp.tools.namefind.BioCodec;
 import opennlp.tools.namefind.NameSample;
 import opennlp.tools.namefind.NameSampleTypeFilter;
 import opennlp.tools.namefind.TokenNameFinderModel;
 import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.SequenceCodec;
 import opennlp.tools.util.model.ArtifactSerializer;
 import opennlp.tools.util.model.ModelUtil;
 
@@ -170,11 +173,22 @@ public final class TokenNameFinderTraine
       sampleStream = new NameSampleTypeFilter(nameTypes, sampleStream);
     }
     
+    String sequenceCodecImplName = params.getSequenceCodec();
+    
+    if ("BIO".equals(sequenceCodecImplName)) {
+      sequenceCodecImplName = BioCodec.class.getName();
+    }
+    else if ("BILOU".equals(sequenceCodecImplName)) {
+      sequenceCodecImplName = BilouCodec.class.getName();
+    }
+    
+    SequenceCodec<String> sequenceCodec = TokenNameFinderModel.instantiateSequenceCodec(sequenceCodecImplName);
+    
     TokenNameFinderModel model;
     try {
       model = opennlp.tools.namefind.NameFinderME.train(
           params.getLang(), params.getType(), sampleStream,
-          mlParams, featureGeneratorBytes, resources);
+          mlParams, featureGeneratorBytes, resources, sequenceCodec);
     }
     catch (IOException e) {
       throw new TerminateToolException(-1, "IO error while reading training data or indexing data: "

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TrainingParams.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TrainingParams.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TrainingParams.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TrainingParams.java Fri Mar  7 10:15:04 2014
@@ -22,6 +22,7 @@ import java.io.File;
 import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
 import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
 import opennlp.tools.cmdline.params.BasicTrainingParams;
+import opennlp.tools.namefind.BioCodec;
 
 /**
  * TrainingParameters for Name Finder.
@@ -45,4 +46,8 @@ interface TrainingParams extends BasicTr
   @OptionalParameter
   @ParameterDescription(valueName = "types", description = "name types to use for training")
   String getNameTypes();
+  
+  @OptionalParameter(defaultValue = "opennlp.tools.namefind.BioCodec")
+  @ParameterDescription(valueName = "codec", description = "sequence codec used to code name spans")
+  String getSequenceCodec();
 }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java Fri Mar  7 10:15:04 2014
@@ -71,7 +71,7 @@ public class NameFinderME implements Tok
   public static final String CONTINUE = "cont";
   public static final String OTHER = "other";
 
-  private static SequenceCodec<String> seqCodec = new BioCodec();
+  private SequenceCodec<String> seqCodec = new BioCodec();
   
   protected SequenceClassificationModel<String> model;
   
@@ -99,6 +99,8 @@ public class NameFinderME implements Tok
   public NameFinderME(TokenNameFinderModel model, AdaptiveFeatureGenerator generator, int beamSize,
       SequenceValidator<String> sequenceValidator) {
     
+    seqCodec = model.createSequenceCodec();
+    
     this.sequenceValidator = sequenceValidator;
    
     // TODO: getNameFinderModel should be removed! Instead the model should always return
@@ -303,9 +305,9 @@ public class NameFinderME implements Tok
     * @throws IOException
     */
    public static TokenNameFinderModel train(String languageCode, String type, ObjectStream<NameSample> samples,
-       TrainingParameters trainParams, AdaptiveFeatureGenerator generator, final Map<String, Object> resources) throws IOException {
-
-     // SequenceCodec seqCodec = new BiolouCodec();
+       TrainingParameters trainParams, AdaptiveFeatureGenerator generator, final Map<String, Object> resources,
+       SequenceCodec<String> seqCodec) throws IOException {
+     
      String beamSizeString = trainParams.getSettings().get(BeamSearch.BEAM_SIZE_PARAMETER);
      
      int beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
@@ -357,17 +359,26 @@ public class NameFinderME implements Tok
        throw new IllegalStateException("Unexpected trainer type!");
      }
      
+     // TODO: Pass the sequence codec down to the model! We will just store the class
+     // name in the model, and then always use the extension loader to create it!
+     // The cmd line interface, will replace shortcuts with actual class names.
+     
      // depending on which one is not null!
      if (seqModel != null) {
        return new TokenNameFinderModel(languageCode, seqModel, null,
-           resources, manifestInfoEntries);
+           resources, manifestInfoEntries, seqCodec);
      }
      else {
        return new TokenNameFinderModel(languageCode, nameFinderModel, beamSize, null,
-           resources, manifestInfoEntries);
+           resources, manifestInfoEntries, seqCodec);
      }
    }
 
+   public static TokenNameFinderModel train(String languageCode, String type, ObjectStream<NameSample> samples,
+       TrainingParameters trainParams, AdaptiveFeatureGenerator generator, final Map<String, Object> resources) throws IOException {
+     return train(languageCode, type, samples, trainParams, generator, resources, new BioCodec());
+   }
+   
   /**
    * Trains a name finder model.
    *
@@ -390,11 +401,11 @@ public class NameFinderME implements Tok
    */
   public static TokenNameFinderModel train(String languageCode, String type,
       ObjectStream<NameSample> samples, TrainingParameters trainParams,
-      byte[] featureGeneratorBytes, final Map<String, Object> resources)
+      byte[] featureGeneratorBytes, final Map<String, Object> resources, SequenceCodec<String> seqCodec)
       throws IOException {
 
     TokenNameFinderModel model = train(languageCode, type, samples, trainParams,
-        createFeatureGenerator(featureGeneratorBytes, resources), resources);
+        createFeatureGenerator(featureGeneratorBytes, resources), resources, seqCodec);
 
     if (featureGeneratorBytes != null) {
       model = model.updateFeatureGenerator(featureGeneratorBytes);
@@ -403,6 +414,13 @@ public class NameFinderME implements Tok
     return model;
   }
 
+  public static TokenNameFinderModel train(String languageCode, String type,
+      ObjectStream<NameSample> samples, TrainingParameters trainParams,
+      byte[] featureGeneratorBytes, final Map<String, Object> resources)
+      throws IOException {
+    return train(languageCode, type, samples, trainParams, featureGeneratorBytes,
+        resources, new BioCodec());
+  }
 
    public static TokenNameFinderModel train(String languageCode, String type, ObjectStream<NameSample> samples,
        final Map<String, Object> resources) throws IOException {

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java Fri Mar  7 10:15:04 2014
@@ -27,6 +27,7 @@ import java.util.Map;
 
 import opennlp.tools.util.FilterObjectStream;
 import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.SequenceCodec;
 import opennlp.tools.util.TrainingParameters;
 import opennlp.tools.util.eval.CrossValidationPartitioner;
 import opennlp.tools.util.eval.FMeasure;
@@ -143,6 +144,7 @@ public class TokenNameFinderCrossValidat
   
 
   private FMeasure fmeasure = new FMeasure();
+  private SequenceCodec<String> codec;
 
   /**
    * Name finder cross validator
@@ -162,7 +164,7 @@ public class TokenNameFinderCrossValidat
    */
   public TokenNameFinderCrossValidator(String languageCode, String type,
       TrainingParameters trainParams, byte[] featureGeneratorBytes,
-      Map<String, Object> resources,
+      Map<String, Object> resources, SequenceCodec<String> codec,
       TokenNameFinderEvaluationMonitor... listeners) {
     
     this.languageCode = languageCode;
@@ -173,8 +175,15 @@ public class TokenNameFinderCrossValidat
     this.params = trainParams;
     
     this.listeners = listeners;
+    this.codec = codec; 
   }
 
+  public TokenNameFinderCrossValidator(String languageCode, String type,
+      TrainingParameters trainParams, byte[] featureGeneratorBytes,
+      Map<String, Object> resources,
+      TokenNameFinderEvaluationMonitor... listeners) {
+    this(languageCode, type, trainParams, featureGeneratorBytes, resources, new BioCodec(), listeners);
+  }
   /**
    * Starts the evaluation.
    * 
@@ -198,7 +207,7 @@ public class TokenNameFinderCrossValidat
           .next();
 
       TokenNameFinderModel model  = opennlp.tools.namefind.NameFinderME.train(languageCode, type,
-            new DocumentToNameSampleStream(trainingSampleStream), params, featureGeneratorBytes, resources);
+            new DocumentToNameSampleStream(trainingSampleStream), params, featureGeneratorBytes, resources, codec);
 
       // do testing
       TokenNameFinderEvaluator evaluator = new TokenNameFinderEvaluator(

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java?rev=1575219&r1=1575218&r2=1575219&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java Fri Mar  7 10:15:04 2014
@@ -24,9 +24,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -34,6 +32,8 @@ import opennlp.tools.ml.BeamSearch;
 import opennlp.tools.ml.model.MaxentModel;
 import opennlp.tools.ml.model.SequenceClassificationModel;
 import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.SequenceCodec;
+import opennlp.tools.util.ext.ExtensionLoader;
 import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
 import opennlp.tools.util.featuregen.AggregatedFeatureGenerator;
 import opennlp.tools.util.featuregen.FeatureGeneratorResourceProvider;
@@ -49,6 +49,7 @@ import opennlp.tools.util.model.ModelUti
  *
  * @see NameFinderME
  */
+// TODO: Fix the model validation, on loading via constructors and input streams
 public class TokenNameFinderModel extends BaseModel {
 
   public static class FeatureGeneratorCreationError extends RuntimeException {
@@ -74,38 +75,42 @@ public class TokenNameFinderModel extend
   private static final String MAXENT_MODEL_ENTRY_NAME = "nameFinder.model";
  
   private static final String GENERATOR_DESCRIPTOR_ENTRY_NAME = "generator.featuregen";
- 
-  public TokenNameFinderModel(String languageCode, SequenceClassificationModel nameFinderModel,
-      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
+
+  private static final String SEQUENCE_CODEC_CLASS_NAME_PARAMETER = "sequenceCodecImplName";
+
+  public TokenNameFinderModel(String languageCode, SequenceClassificationModel<String> nameFinderModel,
+      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
+      SequenceCodec<String> seqCodec) {
     super(COMPONENT_NAME, languageCode, manifestInfoEntries);
     
-    // TODO: Add validation for sequence models!
-    //if (!isModelValid(nameFinderModel)) {
-    //  throw new IllegalArgumentException("Model not compatible with name finder!");
-    //}
+    init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries, seqCodec);
     
-    init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries);
+    if (!seqCodec.areOutcomesCompatible(nameFinderModel.getOutcomes())) {
+      throw new IllegalArgumentException("Model not compatible with name finder!");
+    }
   }
 
   public TokenNameFinderModel(String languageCode, MaxentModel nameFinderModel, int beamSize,
-      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
+      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
+      SequenceCodec<String> seqCodec) {
     super(COMPONENT_NAME, languageCode, manifestInfoEntries);
     
-    if (!isModelValid(nameFinderModel)) {
-      throw new IllegalArgumentException("Model not compatible with name finder!");
-    }
     
     Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
     manifest.put(BeamSearch.BEAM_SIZE_PARAMETER, Integer.toString(beamSize));
     
-    init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries);
+    init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries, seqCodec);
+    
+    if (!isModelValid(nameFinderModel)) {
+      throw new IllegalArgumentException("Model not compatible with name finder!");
+    }
   }
   
   // TODO: Extend this one with beam size!
   public TokenNameFinderModel(String languageCode, MaxentModel nameFinderModel,
       byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
     this(languageCode, nameFinderModel, NameFinderME.DEFAULT_BEAM_SIZE, 
-        generatorDescriptor, resources, manifestInfoEntries);
+        generatorDescriptor, resources, manifestInfoEntries, new BioCodec());
   }
 
   public TokenNameFinderModel(String languageCode, MaxentModel nameFinderModel,
@@ -126,7 +131,12 @@ public class TokenNameFinderModel extend
   }
   
   private void init(Object nameFinderModel,
-      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
+      byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
+      SequenceCodec<String> seqCodec) {
+    
+    Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
+    manifest.put(SEQUENCE_CODEC_CLASS_NAME_PARAMETER, seqCodec.getClass().getName());
+    
     artifactMap.put(MAXENT_MODEL_ENTRY_NAME, nameFinderModel);
     
     if (generatorDescriptor != null && generatorDescriptor.length > 0)
@@ -183,6 +193,16 @@ public class TokenNameFinderModel extend
     }
   }
   
+  public SequenceCodec<String> createSequenceCodec() {
+    
+    // TODO: Lookup impl name with
+    // SEQUENCE_CODEC_CLASS_NAME_PARAMETER
+    Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
+    
+    String sequeceCodecImplName = manifest.getProperty(SEQUENCE_CODEC_CLASS_NAME_PARAMETER);
+    return instantiateSequenceCodec(sequeceCodecImplName);
+  }
+  
   /**
    * Creates the {@link AdaptiveFeatureGenerator}. Usually this
    * is a set of generators contained in the {@link AggregatedFeatureGenerator}.
@@ -235,16 +255,16 @@ public class TokenNameFinderModel extend
         
     TokenNameFinderModel model;
         
-        if (getNameFinderModel() != null) {
-          model = new TokenNameFinderModel(getLanguage(), getNameFinderModel(),
-              descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap());
-        }
-        else {
-          model = new TokenNameFinderModel(getLanguage(), getNameFinderSequenceModel(),
-              descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap());
-        }
+    if (getNameFinderModel() != null) {
+      model = new TokenNameFinderModel(getLanguage(), getNameFinderModel(), 1,
+          descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(), createSequenceCodec());
+    }
+    else {
+      model = new TokenNameFinderModel(getLanguage(), getNameFinderSequenceModel(),
+          descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(),
+          createSequenceCodec());
+    }
     
-    // TODO: Not so nice!
     model.artifactMap.clear();
     model.artifactMap.putAll(artifactMap);
     model.artifactMap.put(GENERATOR_DESCRIPTOR_ENTRY_NAME, descriptor);
@@ -276,44 +296,15 @@ public class TokenNameFinderModel extend
     return serializers;
   }
   
-  // TODO: Write test for this method
-  public static boolean isModelValid(MaxentModel model) {
+  public boolean isModelValid(MaxentModel model) {
+    
+    String outcomes[] = new String[model.getNumOutcomes()];
     
-    // We should have *optionally* one outcome named "other", some named xyz-start and sometimes 
-    // they have a pair xyz-cont. We should not have any other outcome
-    // To validate the model we check if we have one outcome named "other", at least
-    // one outcome with suffix start. After that we check if all outcomes that ends with
-    // "cont" have a pair that ends with "start".
-    List<String> start = new ArrayList<String>();
-    List<String> cont = new ArrayList<String>();
-
     for (int i = 0; i < model.getNumOutcomes(); i++) {
-      String outcome = model.getOutcome(i);
-      if (outcome.endsWith(NameFinderME.START)) {
-        start.add(outcome.substring(0, outcome.length()
-            - NameFinderME.START.length()));
-      } else if (outcome.endsWith(NameFinderME.CONTINUE)) {
-        cont.add(outcome.substring(0, outcome.length()
-            - NameFinderME.CONTINUE.length()));
-      } else if (outcome.equals(NameFinderME.OTHER)) {
-        // don't fail anymore if couldn't find outcome named OTHER
-      } else {
-        // got unexpected outcome
-        return false;
-      }
-    }
-
-    if (start.size() == 0) {
-      return false;
-    } else {
-      for (String contPreffix : cont) {
-        if (!start.contains(contPreffix)) {
-          return false;
-        }
-      }
+      outcomes[i] = model.getOutcome(i);
     }
-
-    return true;
+    
+    return createSequenceCodec().areOutcomesCompatible(outcomes);
   }
   
   @Override
@@ -330,4 +321,17 @@ public class TokenNameFinderModel extend
       throw new InvalidFormatException("Token Name Finder model is incomplete!");
     }
   }
+
+  public static SequenceCodec<String> instantiateSequenceCodec(
+      String sequenceCodecImplName) {
+    
+    if (sequenceCodecImplName != null) {
+      return ExtensionLoader.instantiateExtension(
+          SequenceCodec.class, sequenceCodecImplName);
+    }
+    else {
+      // If nothing is specified return old default!
+      return new BioCodec();
+    }
+  }
 }