You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/05/02 14:34:33 UTC

svn commit: r1591889 [7/14] - in /opennlp/trunk: opennlp-tools/lang/ml/ opennlp-tools/src/main/java/opennlp/tools/chunker/ opennlp-tools/src/main/java/opennlp/tools/cmdline/ opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ opennlp-tools/src/m...

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java Fri May  2 12:34:23 2014
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
- 
+
  package opennlp.tools.namefind;
 
 import java.io.IOException;
@@ -34,21 +34,21 @@ public class NameSampleSequenceStream im
   private final boolean useOutcomes;
   private ObjectStream<NameSample> psi;
   private SequenceCodec<String> seqCodec;
-  
+
   public NameSampleSequenceStream(ObjectStream<NameSample> psi) throws IOException {
     this(psi, new DefaultNameContextGenerator((AdaptiveFeatureGenerator) null), true);
   }
-  
-  public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen) 
+
+  public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen)
   throws IOException {
     this(psi, new DefaultNameContextGenerator(featureGen), true);
   }
 
-  public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen, boolean useOutcomes) 
+  public NameSampleSequenceStream(ObjectStream<NameSample> psi, AdaptiveFeatureGenerator featureGen, boolean useOutcomes)
   throws IOException {
     this(psi, new DefaultNameContextGenerator(featureGen), useOutcomes);
   }
-  
+
   public NameSampleSequenceStream(ObjectStream<NameSample> psi, NameContextGenerator pcg)
       throws IOException {
     this(psi, pcg, true);
@@ -58,7 +58,7 @@ public class NameSampleSequenceStream im
       throws IOException {
     this(psi, pcg, useOutcomes, new BioCodec());
   }
-  
+
   public NameSampleSequenceStream(ObjectStream<NameSample> psi, NameContextGenerator pcg, boolean useOutcomes,
       SequenceCodec<String> seqCodec)
       throws IOException {
@@ -67,7 +67,7 @@ public class NameSampleSequenceStream im
     this.pcg = pcg;
     this.seqCodec = seqCodec;
   }
-  
+
   @SuppressWarnings("unchecked")
   public Event[] updateContext(Sequence sequence, AbstractModel model) {
     Sequence<NameSample> pss = sequence;
@@ -75,12 +75,12 @@ public class NameSampleSequenceStream im
     String[] sentence = pss.getSource().getSentence();
     String[] tags = seqCodec.encode(tagger.find(sentence), sentence.length);
     Event[] events = new Event[sentence.length];
-    
+
     NameFinderEventStream.generateEvents(sentence,tags,pcg).toArray(events);
-    
+
     return events;
   }
-  
+
   @Override
   public Sequence read() throws IOException {
     NameSample sample = psi.read();
@@ -88,7 +88,7 @@ public class NameSampleSequenceStream im
       String sentence[] = sample.getSentence();
       String tags[] = seqCodec.encode(sample.getNames(), sentence.length);
       Event[] events = new Event[sentence.length];
-      
+
       for (int i=0; i < sentence.length; i++) {
 
         // it is safe to pass the tags as previous tags because
@@ -100,7 +100,7 @@ public class NameSampleSequenceStream im
         else {
           context = pcg.getContext(i, sentence, null, null);
         }
-        
+
         events[i] = new Event(tags[i], context);
       }
       Sequence<NameSample> sequence = new Sequence<NameSample>(events,sample);
@@ -110,12 +110,12 @@ public class NameSampleSequenceStream im
         return null;
       }
   }
-  
+
   @Override
   public void reset() throws IOException, UnsupportedOperationException {
     psi.reset();
   }
-  
+
   @Override
   public void close() throws IOException {
     psi.close();

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java Fri May  2 12:34:23 2014
@@ -40,26 +40,26 @@ public class NameSampleTypeFilter extend
     super(samples);
     this.types = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(types)));
   }
-  
+
   public NameSampleTypeFilter(Set<String> types, ObjectStream<NameSample> samples) {
     super(samples);
     this.types = Collections.unmodifiableSet(new HashSet<String>(types));
   }
 
   public NameSample read() throws IOException {
-    
+
     NameSample sample = samples.read();
-    
+
     if (sample != null) {
-      
+
       List<Span> filteredNames = new ArrayList<Span>();
-      
+
       for (Span name : sample.getNames()) {
         if (types.contains(name.getType())) {
           filteredNames.add(name);
         }
       }
-      
+
       return new NameSample(sample.getId(), sample.getSentence(),
           filteredNames.toArray(new Span[filteredNames.size()]), null, sample.isClearAdaptiveDataSet());
     }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinder.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinder.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinder.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinder.java Fri May  2 12:34:23 2014
@@ -53,7 +53,7 @@ public final class RegexNameFinder imple
 
   /**
    * use constructor {@link #RegexNameFinder(Pattern[], String)}
-   * for single types, and/or constructor 
+   * for single types, and/or constructor
    * {@link #RegexNameFinder(Map)}
    */
   @Deprecated

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java Fri May  2 12:34:23 2014
@@ -29,7 +29,7 @@ public interface TokenNameFinder {
    * @return an array of spans for each of the names identified.
    */
   public Span[] find(String tokens[]);
-  
+
   /**
    * Forgets all adaptive data which was collected during previous
    * calls to one of the find methods.

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java Fri May  2 12:34:23 2014
@@ -35,75 +35,75 @@ import opennlp.tools.util.eval.FMeasure;
 public class TokenNameFinderCrossValidator {
 
   private class DocumentSample {
-    
+
     private NameSample samples[];
-    
+
     DocumentSample(NameSample samples[]) {
       this.samples = samples;
     }
-    
+
     private NameSample[] getSamples() {
       return samples;
     }
   }
-  
+
   /**
    * Reads Name Samples to group them as a document based on the clear adaptive data flag.
    */
   private class NameToDocumentSampleStream extends FilterObjectStream<NameSample, DocumentSample> {
 
     private NameSample beginSample;
-    
+
     protected NameToDocumentSampleStream(ObjectStream<NameSample> samples) {
       super(samples);
     }
 
     public DocumentSample read() throws IOException {
-      
+
       List<NameSample> document = new ArrayList<NameSample>();
-      
+
       if (beginSample == null) {
         // Assume that the clear flag is set
         beginSample = samples.read();
       }
-      
-      // Underlying stream is exhausted! 
+
+      // Underlying stream is exhausted!
       if (beginSample == null) {
         return null;
       }
-      
+
       document.add(beginSample);
-      
+
       NameSample sample;
       while ((sample = samples.read()) != null) {
-        
+
         if (sample.isClearAdaptiveDataSet()) {
           beginSample = sample;
           break;
         }
-        
+
         document.add(sample);
       }
-      
+
       // Underlying stream is exhausted,
       // next call must return null
       if (sample == null) {
         beginSample = null;
       }
-      
+
       return new DocumentSample(document.toArray(new NameSample[document.size()]));
     }
-    
+
     @Override
     public void reset() throws IOException, UnsupportedOperationException {
       super.reset();
-      
+
       beginSample = null;
     }
   }
-  
+
   /**
-   * Splits DocumentSample into NameSamples. 
+   * Splits DocumentSample into NameSamples.
    */
   private class DocumentToNameSampleStream extends FilterObjectStream<DocumentSample, NameSample>{
 
@@ -148,7 +148,7 @@ public class TokenNameFinderCrossValidat
 
   /**
    * Name finder cross validator
-   * 
+   *
    * @param languageCode
    *          the language of the training data
    * @param type
@@ -166,16 +166,16 @@ public class TokenNameFinderCrossValidat
       TrainingParameters trainParams, byte[] featureGeneratorBytes,
       Map<String, Object> resources, SequenceCodec<String> codec,
       TokenNameFinderEvaluationMonitor... listeners) {
-    
+
     this.languageCode = languageCode;
     this.type = type;
     this.featureGeneratorBytes = featureGeneratorBytes;
     this.resources = resources;
 
     this.params = trainParams;
-    
+
     this.listeners = listeners;
-    this.codec = codec; 
+    this.codec = codec;
   }
 
   public TokenNameFinderCrossValidator(String languageCode, String type,
@@ -184,7 +184,7 @@ public class TokenNameFinderCrossValidat
       TokenNameFinderEvaluationMonitor... listeners) {
     this(languageCode, type, trainParams, featureGeneratorBytes, resources, new BioCodec(), listeners);
   }
-  
+
   public TokenNameFinderCrossValidator(String languageCode, String type,
       TrainingParameters trainParams, TokenNameFinderFactory factory,
       TokenNameFinderEvaluationMonitor... listeners) {
@@ -197,7 +197,7 @@ public class TokenNameFinderCrossValidat
 
   /**
    * Starts the evaluation.
-   * 
+   *
    * @param samples
    *          the data to train and test
    * @param nFolds
@@ -224,7 +224,7 @@ public class TokenNameFinderCrossValidat
       else {
         model  = opennlp.tools.namefind.NameFinderME.train(languageCode, type,
             new DocumentToNameSampleStream(trainingSampleStream), params, featureGeneratorBytes, resources);
-        
+
       }
 
       // do testing

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java Fri May  2 12:34:23 2014
@@ -43,19 +43,19 @@ import opennlp.tools.util.eval.FMeasure;
 public class TokenNameFinderEvaluator extends Evaluator<NameSample> {
 
   private FMeasure fmeasure = new FMeasure();
-  
+
   /**
    * The {@link TokenNameFinder} used to create the predicted
    * {@link NameSample} objects.
    */
   private TokenNameFinder nameFinder;
-  
+
   /**
    * Initializes the current instance with the given
    * {@link TokenNameFinder}.
    *
    * @param nameFinder the {@link TokenNameFinder} to evaluate.
-   * @param listeners evaluation sample listeners 
+   * @param listeners evaluation sample listeners
    */
   public TokenNameFinderEvaluator(TokenNameFinder nameFinder, TokenNameFinderEvaluationMonitor ... listeners) {
     super(listeners);
@@ -71,17 +71,17 @@ public class TokenNameFinderEvaluator ex
    * calculate and update the scores.
    *
    * @param reference the reference {@link NameSample}.
-   * 
+   *
    * @return the predicted {@link NameSample}.
    */
   @Override
   protected NameSample processSample(NameSample reference) {
-    
+
     if (reference.isClearAdaptiveDataSet()) {
       nameFinder.clearAdaptiveData();
     }
-    
-    Span predictedNames[] = nameFinder.find(reference.getSentence());    
+
+    Span predictedNames[] = nameFinder.find(reference.getSentence());
     Span references[] = reference.getNames();
 
     // OPENNLP-396 When evaluating with a file in the old format
@@ -92,59 +92,59 @@ public class TokenNameFinderEvaluator ex
         references[i] = new Span(references[i].getStart(), references[i].getEnd(), "default");
       }
     }
-    
+
     fmeasure.updateScores(references, predictedNames);
-    
+
     return new NameSample(reference.getSentence(), predictedNames, reference.isClearAdaptiveDataSet());
   }
-  
+
   public FMeasure getFMeasure() {
     return fmeasure;
   }
-  
+
   @Deprecated
-  public static void main(String[] args) throws IOException, 
+  public static void main(String[] args) throws IOException,
       InvalidFormatException {
-    
+
     if (args.length == 4) {
-      
+
       System.out.println("Loading name finder model ...");
       InputStream modelIn = new FileInputStream(args[3]);
-      
+
       TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
-      
+
       TokenNameFinder nameFinder = new NameFinderME(model);
-      
+
       System.out.println("Performing evaluation ...");
       TokenNameFinderEvaluator evaluator = new TokenNameFinderEvaluator(nameFinder);
-      
+
       final NameSampleDataStream sampleStream = new NameSampleDataStream(
           new PlainTextByLineStream(new InputStreamReader(new FileInputStream(args[2]), args[1])));
-      
+
       final PerformanceMonitor monitor = new PerformanceMonitor("sent");
-      
+
       monitor.startAndPrintThroughput();
-      
+
       ObjectStream<NameSample> iterator = new ObjectStream<NameSample>() {
 
         public NameSample read() throws IOException {
           monitor.incrementCounter();
           return sampleStream.read();
         }
-        
+
         public void reset() throws IOException {
           sampleStream.reset();
         }
-        
+
         public void close() throws IOException {
           sampleStream.close();
         }
       };
-      
+
       evaluator.evaluate(iterator);
-      
+
       monitor.stopAndPrintFinalResult();
-      
+
       System.out.println();
       System.out.println("F-Measure: " + evaluator.getFMeasure().getFMeasure());
       System.out.println("Recall: " + evaluator.getFMeasure().getRecallScore());

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java Fri May  2 12:34:23 2014
@@ -56,7 +56,7 @@ public class TokenNameFinderFactory exte
   public TokenNameFinderFactory() {
     this.seqCodec = new BioCodec();
   }
-  
+
   public TokenNameFinderFactory(byte[] featureGeneratorBytes, final Map<String, Object> resources,
       SequenceCodec<String> seqCodec) {
     init(featureGeneratorBytes, resources, seqCodec);
@@ -67,15 +67,15 @@ public class TokenNameFinderFactory exte
     this.resources = resources;
     this.seqCodec = seqCodec;
   }
-  
+
   protected SequenceCodec<String> getSequenceCodec() {
     return seqCodec;
   }
-  
+
   protected Map<String, Object> getResources() {
     return resources;
   }
-  
+
   public static TokenNameFinderFactory create(String subclassName, byte[] featureGeneratorBytes, final Map<String, Object> resources,
       SequenceCodec<String> seqCodec)
       throws InvalidFormatException {
@@ -101,9 +101,9 @@ public class TokenNameFinderFactory exte
   public void validateArtifactMap() throws InvalidFormatException {
     // no additional artifacts
   }
-  
+
   public SequenceCodec<String> createSequenceCodec() {
-    
+
     if (artifactProvider != null) {
       String sequeceCodecImplName = artifactProvider.getManifestProperty(
           TokenNameFinderModel.SEQUENCE_CODEC_CLASS_NAME_PARAMETER);
@@ -115,16 +115,16 @@ public class TokenNameFinderFactory exte
   }
 
   public NameContextGenerator createContextGenerator() {
-    
+
     AdaptiveFeatureGenerator featureGenerator = createFeatureGenerators();
-    
+
     if (featureGenerator == null) {
       featureGenerator = NameFinderME.createFeatureGenerator();
     }
-    
+
     return new DefaultNameContextGenerator(featureGenerator);
   }
-  
+
   /**
    * Creates the {@link AdaptiveFeatureGenerator}. Usually this
    * is a set of generators contained in the {@link AggregatedFeatureGenerator}.
@@ -145,14 +145,14 @@ public class TokenNameFinderFactory exte
     else {
       descriptorBytes = featureGeneratorBytes;
     }
-    
+
     if (descriptorBytes != null) {
       InputStream descriptorIn = new ByteArrayInputStream(descriptorBytes);
-  
+
       AdaptiveFeatureGenerator generator = null;
       try {
         generator = GeneratorFactory.create(descriptorIn, new FeatureGeneratorResourceProvider() {
-  
+
           public Object getResource(String key) {
             if (artifactProvider != null) {
               return artifactProvider.getArtifact(key);
@@ -165,30 +165,30 @@ public class TokenNameFinderFactory exte
       } catch (InvalidFormatException e) {
         // It is assumed that the creation of the feature generation does not
         // fail after it succeeded once during model loading.
-        
+
         // But it might still be possible that such an exception is thrown,
         // in this case the caller should not be forced to handle the exception
         // and a Runtime Exception is thrown instead.
-        
+
         // If the re-creation of the feature generation fails it is assumed
         // that this can only be caused by a programming mistake and therefore
         // throwing a Runtime Exception is reasonable
-        
+
         throw new FeatureGeneratorCreationError(e);
       } catch (IOException e) {
         throw new IllegalStateException("Reading from mem cannot result in an I/O error", e);
       }
-  
+
       return generator;
     }
     else {
       return null;
     }
   }
-  
+
   public static SequenceCodec<String> instantiateSequenceCodec(
       String sequenceCodecImplName) {
-    
+
     if (sequenceCodecImplName != null) {
       return ExtensionLoader.instantiateExtension(
           SequenceCodec.class, sequenceCodecImplName);

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java Fri May  2 12:34:23 2014
@@ -59,12 +59,12 @@ public class TokenNameFinderModel extend
       super(t);
     }
   }
-  
+
   private static class ByteArraySerializer implements ArtifactSerializer<byte[]> {
 
     public byte[] create(InputStream in) throws IOException,
         InvalidFormatException {
-      
+
       return ModelUtil.read(in);
     }
 
@@ -72,10 +72,10 @@ public class TokenNameFinderModel extend
       out.write(artifact);
     }
   }
-  
+
   private static final String COMPONENT_NAME = "NameFinderME";
   private static final String MAXENT_MODEL_ENTRY_NAME = "nameFinder.model";
- 
+
   static final String GENERATOR_DESCRIPTOR_ENTRY_NAME = "generator.featuregen";
 
   static final String SEQUENCE_CODEC_CLASS_NAME_PARAMETER = "sequenceCodecImplName";
@@ -84,9 +84,9 @@ public class TokenNameFinderModel extend
       byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
       SequenceCodec<String> seqCodec) {
     super(COMPONENT_NAME, languageCode, manifestInfoEntries);
-    
+
     init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries, seqCodec);
-    
+
     if (!seqCodec.areOutcomesCompatible(nameFinderModel.getOutcomes())) {
       throw new IllegalArgumentException("Model not compatible with name finder!");
     }
@@ -96,22 +96,22 @@ public class TokenNameFinderModel extend
       byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
       SequenceCodec<String> seqCodec) {
     super(COMPONENT_NAME, languageCode, manifestInfoEntries);
-    
-    
+
+
     Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
     manifest.put(BeamSearch.BEAM_SIZE_PARAMETER, Integer.toString(beamSize));
-    
+
     init(nameFinderModel, generatorDescriptor, resources, manifestInfoEntries, seqCodec);
-    
+
     if (!isModelValid(nameFinderModel)) {
       throw new IllegalArgumentException("Model not compatible with name finder!");
     }
   }
-  
+
   // TODO: Extend this one with beam size!
   public TokenNameFinderModel(String languageCode, MaxentModel nameFinderModel,
       byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
-    this(languageCode, nameFinderModel, NameFinderME.DEFAULT_BEAM_SIZE, 
+    this(languageCode, nameFinderModel, NameFinderME.DEFAULT_BEAM_SIZE,
         generatorDescriptor, resources, manifestInfoEntries, new BioCodec());
   }
 
@@ -119,31 +119,31 @@ public class TokenNameFinderModel extend
       Map<String, Object> resources, Map<String, String> manifestInfoEntries) {
     this(languageCode, nameFinderModel, null, resources, manifestInfoEntries);
   }
-      
+
   public TokenNameFinderModel(InputStream in) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, in);
   }
-  
+
   public TokenNameFinderModel(File modelFile) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelFile);
   }
-  
+
   public TokenNameFinderModel(URL modelURL) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelURL);
   }
-  
+
   private void init(Object nameFinderModel,
       byte[] generatorDescriptor, Map<String, Object> resources, Map<String, String> manifestInfoEntries,
       SequenceCodec<String> seqCodec) {
-    
+
     Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
     manifest.put(SEQUENCE_CODEC_CLASS_NAME_PARAMETER, seqCodec.getClass().getName());
-    
+
     artifactMap.put(MAXENT_MODEL_ENTRY_NAME, nameFinderModel);
-    
+
     if (generatorDescriptor != null && generatorDescriptor.length > 0)
       artifactMap.put(GENERATOR_DESCRIPTOR_ENTRY_NAME, generatorDescriptor);
-    
+
     if (resources != null) {
       // The resource map must not contain key which are already taken
       // like the name finder maxent model name
@@ -151,20 +151,20 @@ public class TokenNameFinderModel extend
           resources.containsKey(GENERATOR_DESCRIPTOR_ENTRY_NAME)) {
         throw new IllegalArgumentException();
       }
-      
+
       // TODO: Add checks to not put resources where no serializer exists,
       // make that case fail here, should be done in the BaseModel
-      artifactMap.putAll(resources); 
+      artifactMap.putAll(resources);
     }
     checkArtifactMap();
   }
-  
+
   /**
    * @deprecated use getNameFinderSequenceModel instead. This method will be removed soon.
    */
   @Deprecated
   public MaxentModel getNameFinderModel() {
-    
+
     if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel) {
       return (MaxentModel) artifactMap.get(MAXENT_MODEL_ENTRY_NAME);
     }
@@ -174,17 +174,17 @@ public class TokenNameFinderModel extend
   }
 
   public SequenceClassificationModel<String> getNameFinderSequenceModel() {
-    
+
     Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
-    
+
     if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel) {
       String beamSizeString = manifest.getProperty(BeamSearch.BEAM_SIZE_PARAMETER);
-      
+
       int beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
       if (beamSizeString != null) {
         beamSize = Integer.parseInt(beamSizeString);
       }
-      
+
       return new BeamSearch<>(beamSize, (MaxentModel) artifactMap.get(MAXENT_MODEL_ENTRY_NAME));
     }
     else if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
@@ -194,19 +194,19 @@ public class TokenNameFinderModel extend
       return null;
     }
   }
-  
+
   @Override
   protected Class<? extends BaseToolFactory> getDefaultFactory() {
     return TokenNameFinderFactory.class;
   }
-  
+
   public TokenNameFinderFactory getFactory() {
     return (TokenNameFinderFactory) this.toolFactory;
   }
 
   // TODO: This should be moved to the NameFinderFactory ... !!!
   // Lets deprecate it!
-  
+
   /**
    * Creates the {@link AdaptiveFeatureGenerator}. Usually this
    * is a set of generators contained in the {@link AggregatedFeatureGenerator}.
@@ -221,11 +221,11 @@ public class TokenNameFinderModel extend
   public AdaptiveFeatureGenerator createFeatureGenerators() {
     return getFactory().createFeatureGenerators();
   }
-  
+
   public TokenNameFinderModel updateFeatureGenerator(byte descriptor[]) {
-        
+
     TokenNameFinderModel model;
-        
+
     if (getNameFinderModel() != null) {
       model = new TokenNameFinderModel(getLanguage(), getNameFinderModel(), 1,
           descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(),
@@ -236,53 +236,53 @@ public class TokenNameFinderModel extend
           descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(),
           getFactory().createSequenceCodec());
     }
-    
+
     model.artifactMap.clear();
     model.artifactMap.putAll(artifactMap);
     model.artifactMap.put(GENERATOR_DESCRIPTOR_ENTRY_NAME, descriptor);
-    
+
     return model;
   }
-  
+
   @Override
   protected void createArtifactSerializers(Map<String, ArtifactSerializer> serializers) {
     super.createArtifactSerializers(serializers);
-    
+
     serializers.put("featuregen", new ByteArraySerializer());
   }
-  
+
   public static Map<String, ArtifactSerializer> createArtifactSerializers()  {
-    
+
     // TODO: Not so nice, because code cannot really be reused by the other create serializer method
     //       Has to be redesigned, we need static access to default serializers
-    //       and these should be able to extend during runtime ?! 
+    //       and these should be able to extend during runtime ?!
     //
     //       The XML feature generator factory should provide these mappings.
     //       Usually the feature generators should know what type of resource they expect.
-    
+
     Map<String, ArtifactSerializer> serializers = BaseModel.createArtifactSerializers();
-    
+
     serializers.put("featuregen", new ByteArraySerializer());
     serializers.put("w2vclasses", new W2VClassesDictionary.W2VClassesDictionarySerializer());
-    
+
     return serializers;
   }
-  
+
   boolean isModelValid(MaxentModel model) {
-    
+
     String outcomes[] = new String[model.getNumOutcomes()];
-    
+
     for (int i = 0; i < model.getNumOutcomes(); i++) {
       outcomes[i] = model.getOutcome(i);
     }
-    
+
     return getFactory().createSequenceCodec().areOutcomesCompatible(outcomes);
   }
-  
+
   @Override
   protected void validateArtifactMap() throws InvalidFormatException {
     super.validateArtifactMap();
-    
+
     if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel ||
         artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
       // TODO: Check should be performed on the possible outcomes!

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java Fri May  2 12:34:23 2014
@@ -23,7 +23,7 @@ import java.util.List;
  * of strings
  */
 public class NGramGenerator {
- 
+
 
   /**
    * Creates an ngram separated

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java Fri May  2 12:34:23 2014
@@ -48,52 +48,52 @@ import opennlp.tools.util.TrainingParame
  */
 public abstract class AbstractBottomUpParser implements Parser {
 
-  /** 
+  /**
    * The maximum number of parses advanced from all preceding
    * parses at each derivation step.
    */
   protected int M;
-  
-  /** 
+
+  /**
    * The maximum number of parses to advance from a single preceding parse.
    */
   protected int K;
-  
-  /** 
+
+  /**
    * The minimum total probability mass of advanced outcomes.
    */
   protected double Q;
-  
+
   /**
    * The default beam size used if no beam size is given.
    */
   public static final int defaultBeamSize = 20;
-  
-  /** 
+
+  /**
    * The default amount of probability mass required of advanced outcomes.
    */
   public static final double defaultAdvancePercentage = 0.95;
-  
-  /** 
+
+  /**
    * Completed parses.
    */
   protected Heap<Parse> completeParses;
-  
-  /** 
+
+  /**
    * Incomplete parses which will be advanced.
    */
   protected Heap<Parse> odh;
-  
-  /** 
-   * Incomplete parses which have been advanced. 
+
+  /**
+   * Incomplete parses which have been advanced.
    */
   protected Heap<Parse> ndh;
 
-  /** 
+  /**
    * The head rules for the parser.
    */
   protected HeadRules headRules;
-  
+
   /**
    * The set strings which are considered punctuation for the parser.
    * Punctuation is not attached, but floats to the top of the parse as attachment
@@ -101,73 +101,73 @@ public abstract class AbstractBottomUpPa
    */
   protected Set<String> punctSet;
 
-  /** 
-   * The label for the top node. 
+  /**
+   * The label for the top node.
    */
   public static final String TOP_NODE = "TOP";
-  
-  /** 
+
+  /**
    * The label for the top if an incomplete node.
    */
   public static final String INC_NODE = "INC";
-  
-  /** 
-   * The label for a token node. 
+
+  /**
+   * The label for a token node.
    */
   public static final String TOK_NODE = "TK";
-  
-  /** 
+
+  /**
    * The integer 0.
    */
   public static final Integer ZERO = 0;
 
-  /** 
-   * Prefix for outcomes starting a constituent. 
+  /**
+   * Prefix for outcomes starting a constituent.
    */
   public static final String START = "S-";
-  
-  /** 
-   * Prefix for outcomes continuing a constituent. 
+
+  /**
+   * Prefix for outcomes continuing a constituent.
    */
   public static final String CONT = "C-";
-  
-  /** 
-   * Outcome for token which is not contained in a basal constituent. 
+
+  /**
+   * Outcome for token which is not contained in a basal constituent.
    */
   public static final String OTHER = "O";
-  
-  /** 
-   * Outcome used when a constituent is complete. 
+
+  /**
+   * Outcome used when a constituent is complete.
    */
   public static final String COMPLETE = "c";
-  
-  /** 
-   * Outcome used when a constituent is incomplete. 
+
+  /**
+   * Outcome used when a constituent is incomplete.
    */
   public static final String INCOMPLETE = "i";
 
-  /** 
-   * The pos-tagger that the parser uses. 
+  /**
+   * The pos-tagger that the parser uses.
    */
   protected POSTagger tagger;
 
-  /** 
-   * The chunker that the parser uses to chunk non-recursive structures. 
+  /**
+   * The chunker that the parser uses to chunk non-recursive structures.
    */
   protected Chunker chunker;
 
-  /** 
-   * Specifies whether failed parses should be reported to standard error. 
+  /**
+   * Specifies whether failed parses should be reported to standard error.
    */
   protected boolean reportFailedParse;
 
-  /** 
-   * Specifies whether a derivation string should be created during parsing. 
-   * This is useful for debugging. 
+  /**
+   * Specifies whether a derivation string should be created during parsing.
+   * This is useful for debugging.
    */
   protected boolean createDerivationString = false;
 
-  /** 
+  /**
    * Turns debug print on or off.
    */
   protected boolean debugOn = false;
@@ -248,7 +248,7 @@ public abstract class AbstractBottomUpPa
 
 
 
-  /** 
+  /**
    * Advances the specified parse and returns the an array advanced parses whose probability accounts for
    * more than the specified amount of probability mass.
    * @param p The parse to advance.
@@ -360,7 +360,7 @@ public abstract class AbstractBottomUpPa
   }
 
   public Parse parse(Parse tokens) {
-    
+
     if (tokens.getChildCount() > 0) {
       Parse p = parse(tokens,1)[0];
       setParents(p);
@@ -497,19 +497,19 @@ public abstract class AbstractBottomUpPa
     }
     return parseIndex;
   }
-  
+
   private static boolean lastChild(Parse child, Parse parent, Set<String> punctSet) {
     if (parent == null) {
       return false;
     }
-    
+
     Parse[] kids = collapsePunctuation(parent.getChildren(), punctSet);
     return (kids[kids.length - 1] == child);
   }
-  
+
   /**
    * Creates a n-gram dictionary from the specified data stream using the specified head rule and specified cut-off.
-   * 
+   *
    * @param data The data stream of parses.
    * @param rules The head rules for the parses.
    * @param params can contain a cutoff, the minimum number of entries required for the
@@ -518,17 +518,17 @@ public abstract class AbstractBottomUpPa
    */
   public static Dictionary buildDictionary(ObjectStream<Parse> data, HeadRules rules, TrainingParameters params)
       throws IOException {
-    
+
     int cutoff = 5;
-    
+
     String cutoffString = params.getSettings("dict").
         get(TrainingParameters.CUTOFF_PARAM);
-    
+
     if (cutoffString != null) {
       // TODO: Maybe throw illegal argument exception if not parse able
       cutoff = Integer.parseInt(cutoffString);
     }
-    
+
     NGramModel mdict = new NGramModel();
     Parse p;
     while((p = data.read()) != null) {
@@ -553,7 +553,7 @@ public abstract class AbstractBottomUpPa
       int ci = 0;
       while (ci < chunks.length) {
         //System.err.println("chunks["+ci+"]="+chunks[ci].getHead().getCoveredText()+" chunks.length="+chunks.length + "  " + chunks[ci].getParent());
-        
+
         if (chunks[ci].getParent() == null) {
           chunks[ci].show();
         }
@@ -597,10 +597,10 @@ public abstract class AbstractBottomUpPa
     mdict.cutoff(cutoff, Integer.MAX_VALUE);
     return mdict.toDictionary(true);
   }
-  
+
   /**
    * Creates a n-gram dictionary from the specified data stream using the specified head rule and specified cut-off.
-   * 
+   *
    * @param data The data stream of parses.
    * @param rules The head rules for the parses.
    * @param cutoff The minimum number of entries required for the n-gram to be saved as part of the dictionary.
@@ -608,10 +608,10 @@ public abstract class AbstractBottomUpPa
    */
   public static Dictionary buildDictionary(ObjectStream<Parse> data, HeadRules rules, int cutoff)
       throws IOException {
-    
+
     TrainingParameters params = new TrainingParameters();
     params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cutoff));
-    
+
     return buildDictionary(data, rules, params);
   }
 }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java Fri May  2 12:34:23 2014
@@ -40,8 +40,8 @@ public abstract class AbstractParserEven
   private POSContextGenerator tagContextGenerator;
   protected HeadRules rules;
   protected Set<String> punctSet;
-  
-  /** 
+
+  /**
    * The type of events being generated by this event stream.
    */
   protected ParserEventTypeEnum etype;
@@ -67,7 +67,7 @@ public abstract class AbstractParserEven
   @Override
   protected Iterator<Event> createEvents(Parse sample) {
     List<Event> newEvents = new ArrayList<Event>();
-    
+
     Parse.pruneParse(sample);
     if (fixPossesives) {
       Parse.fixPossesives(sample);
@@ -83,10 +83,10 @@ public abstract class AbstractParserEven
     else {
       addParseEvents(newEvents, Parser.collapsePunctuation(chunks,punctSet));
     }
-    
+
     return newEvents.iterator();
   }
-  
+
   protected void init() {
     fixPossesives = false;
   }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java Fri May  2 12:34:23 2014
@@ -27,7 +27,7 @@ import opennlp.tools.util.FilterObjectSt
 import opennlp.tools.util.ObjectStream;
 
 public class ChunkSampleStream extends FilterObjectStream<Parse, ChunkSample> {
-  
+
   public ChunkSampleStream(ObjectStream<Parse> in) {
     super(in);
   }
@@ -61,11 +61,11 @@ public class ChunkSampleStream extends F
     getInitialChunks(p, chunks);
     return chunks.toArray(new Parse[chunks.size()]);
   }
-  
+
   public ChunkSample read() throws IOException {
-    
+
     Parse parse = samples.read();
-    
+
     if (parse != null) {
       Parse[] chunks = getInitialChunks(parse);
       List<String> toks = new ArrayList<String>();
@@ -96,9 +96,9 @@ public class ChunkSampleStream extends F
           }
         }
       }
-      
-      return new ChunkSample(toks.toArray(new String[toks.size()]), 
-          tags.toArray(new String[tags.size()]), 
+
+      return new ChunkSample(toks.toArray(new String[toks.size()]),
+          tags.toArray(new String[tags.size()]),
           preds.toArray(new String[preds.size()]));
     }
     else {

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java Fri May  2 12:34:23 2014
@@ -44,7 +44,7 @@ public class Parse implements Cloneable,
   public static final String BRACKET_RCB = "}";
   public static final String BRACKET_LSB = "[";
   public static final String BRACKET_RSB = "]";
-  
+
   /**
    * The text string on which this parse is based.
    * This object is shared among all parses for the same sentence.
@@ -433,9 +433,9 @@ public class Parse implements Cloneable,
   public String getCoveredText() {
     return text.substring(span.getStart(), span.getEnd());
   }
-  
+
   /**
-   * Represents this parse in a human readable way.  
+   * Represents this parse in a human readable way.
    */
   @Override
   public String toString() {
@@ -604,7 +604,7 @@ public class Parse implements Cloneable,
     return parts.indexOf(child);
   }
 
-  /** 
+  /**
    * Returns the head constituent associated with this constituent.
    *
    * @return The head constituent associated with this constituent.
@@ -661,7 +661,7 @@ public class Parse implements Cloneable,
     else if (rest.startsWith("-LSB-")) {
       return "-LSB-";
     }
-    
+
     else if (rest.startsWith("-NONE-")) {
       return "-NONE-";
     }
@@ -701,10 +701,10 @@ public class Parse implements Cloneable,
     else if (BRACKET_RSB.equals(token)) {
       return "-RSB-";
     }
-    
+
     return token;
   }
-  
+
   private static String decodeToken(String token) {
     if ("-LRB-".equals(token)) {
       return BRACKET_LRB;
@@ -724,10 +724,10 @@ public class Parse implements Cloneable,
     else if ("-RSB-".equals(token)) {
       return BRACKET_RSB;
     }
-    
+
     return token;
   }
-  
+
   /**
    * Returns the string containing the token for the specified portion of the parse string or
    * null if the portion of the parse string does not represent a token.
@@ -992,7 +992,7 @@ public class Parse implements Cloneable,
     }
     return null;
   }
-  
+
   @Override
   public boolean equals(Object o) {
     if (o instanceof Parse) {
@@ -1023,7 +1023,7 @@ public class Parse implements Cloneable,
     }
     return false;
   }
-  
+
   @Override
   public int hashCode() {
     int result = 17;
@@ -1092,7 +1092,7 @@ public class Parse implements Cloneable,
 
   /**
    * Utility method to inserts named entities.
-   * 
+   *
    * @param tag
    * @param names
    * @param tokens
@@ -1132,7 +1132,7 @@ public class Parse implements Cloneable,
       }
     }
   }
-  
+
   /**
    * Reads training parses (one-sentence-per-line) and displays parse structure.
    *

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java Fri May  2 12:34:23 2014
@@ -29,9 +29,9 @@ public class ParseSampleStream extends F
   }
 
   public Parse read() throws IOException {
-    
+
     String parse = samples.read();
-    
+
     if (parse != null) {
       if (!parse.trim().isEmpty()) {
         return Parse.parseParse(parse);
@@ -39,7 +39,7 @@ public class ParseSampleStream extends F
       else {
         return read();
       }
-    } 
+    }
     else {
       return null;
     }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java Fri May  2 12:34:23 2014
@@ -29,80 +29,80 @@ import opennlp.tools.util.eval.FMeasure;
 public class ParserEvaluator extends Evaluator<Parse> {
 
   private FMeasure fmeasure = new FMeasure();
-  
+
   private final Parser parser;
 
   public ParserEvaluator(Parser parser, ParserEvaluationMonitor... monitors) {
     super(monitors);
     this.parser = parser;
   }
-  
+
   private static Span[] getConstituencySpans(Parse parse) {
-    
+
     Stack<Parse> stack = new Stack<Parse>();
-    
+
     if (parse.getChildCount() > 0) {
       stack.add(parse.getChildren()[0]);
     }
-    
+
     List<Span> consts = new ArrayList<Span>();
-    
+
     while (!stack.isEmpty()) {
-      
+
       Parse constSpan = stack.pop();
-      
+
       if (!constSpan.isPosTag()) {
         Span span = constSpan.getSpan();
         consts.add(new Span(span.getStart(), span.getEnd(), constSpan.getType()));
-        
+
         for (Parse child : constSpan.getChildren()) {
           stack.push(child);
         }
       }
     }
-    
+
     return consts.toArray(new Span[consts.size()]);
   }
-  
+
   @Override
   protected Parse processSample(Parse reference) {
-    
+
     String sentenceText = reference.getText();
-    
+
     Parse predictions[] = ParserTool.parseLine(sentenceText, parser, 1);
-    
+
     Parse prediction = null;
     if (predictions.length > 0) {
       prediction = predictions[0];
     }
-    
+
     fmeasure.updateScores(getConstituencySpans(reference), getConstituencySpans(prediction));
-    
+
     return prediction;
   }
-  
+
   public FMeasure getFMeasure() {
     return fmeasure;
   }
-  
+
   public static void main(String[] args) {
-    
+
     // TODO: Move this to a test case!
-    
+
     String goldParseString = "(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care))) ))  (NP (NN yesterday)) (. .) ))";
     Span goldConsts[] = getConstituencySpans(Parse.parseParse(goldParseString));
-    
+
     String testParseString = "(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care) (NN yesterday))) ))  (. .) ))";
     Span testConsts[] = getConstituencySpans(Parse.parseParse(testParseString));
-    
+
     FMeasure measure = new FMeasure();
     measure.updateScores(goldConsts, testConsts);
-    
+
     // Expected output:
     // Precision: 0.42857142857142855
     // Recall: 0.375
     // F-Measure: 0.39999999999999997
-    
+
     System.out.println(measure.toString());
   }
 }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserFactory.java Fri May  2 12:34:23 2014
@@ -21,9 +21,9 @@ public class ParserFactory {
 
   private ParserFactory() {
   }
-  
+
   public static Parser create(ParserModel model, int beamSize, double advancePercentage) {
-    
+
     if (ParserType.CHUNKING.equals(model.getParserType())) {
       return new opennlp.tools.parser.chunking.Parser(model, beamSize, advancePercentage);
     }
@@ -31,13 +31,13 @@ public class ParserFactory {
       return new opennlp.tools.parser.treeinsert.Parser(model, beamSize, advancePercentage);
     }
     else {
-      throw new IllegalStateException("Unexpected ParserType: " + 
+      throw new IllegalStateException("Unexpected ParserType: " +
           model.getParserType().name());
     }
   }
-  
+
   public static Parser create(ParserModel model) {
-    return create(model, AbstractBottomUpParser.defaultBeamSize, 
+    return create(model, AbstractBottomUpParser.defaultBeamSize,
         AbstractBottomUpParser.defaultAdvancePercentage);
   }
 }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java Fri May  2 12:34:23 2014
@@ -55,7 +55,7 @@ public class ParserModel extends BaseMod
       artifact.serialize(out);
     }
   }
-  
+
   private static class ChunkerModelSerializer implements ArtifactSerializer<ChunkerModel> {
 
     public ChunkerModel create(InputStream in) throws IOException,
@@ -68,7 +68,7 @@ public class ParserModel extends BaseMod
       artifact.serialize(out);
     }
   }
-  
+
   private static class HeadRulesSerializer implements
       ArtifactSerializer<opennlp.tools.parser.lang.en.HeadRules> {
 
@@ -83,13 +83,13 @@ public class ParserModel extends BaseMod
       artifact.serialize(new OutputStreamWriter(out, "UTF-8"));
     }
   }
-  
+
   private static final String COMPONENT_NAME = "Parser";
-  
+
   private static final String BUILD_MODEL_ENTRY_NAME = "build.model";
 
   private static final String CHECK_MODEL_ENTRY_NAME = "check.model";
-  
+
   private static final String ATTACH_MODEL_ENTRY_NAME = "attach.model";
 
   private static final String PARSER_TAGGER_MODEL_ENTRY_NAME = "parsertager.postagger";
@@ -97,20 +97,20 @@ public class ParserModel extends BaseMod
   private static final String CHUNKER_TAGGER_MODEL_ENTRY_NAME = "parserchunker.chunker";
 
   private static final String HEAD_RULES_MODEL_ENTRY_NAME = "head-rules.headrules";
-  
+
   private static final String PARSER_TYPE = "parser-type";
-  
-  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel, 
+
+  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel,
       MaxentModel attachModel, POSModel parserTagger,
       ChunkerModel chunkerTagger, opennlp.tools.parser.HeadRules headRules,
       ParserType modelType, Map<String, String> manifestInfoEntries) {
 
     super(COMPONENT_NAME, languageCode, manifestInfoEntries);
-    
+
     setManifestProperty(PARSER_TYPE, modelType.name());
-    
+
     artifactMap.put(BUILD_MODEL_ENTRY_NAME, buildModel);
-    
+
     artifactMap.put(CHECK_MODEL_ENTRY_NAME, checkModel);
 
     if (ParserType.CHUNKING.equals(modelType)) {
@@ -120,73 +120,73 @@ public class ParserModel extends BaseMod
     else if (ParserType.TREEINSERT.equals(modelType)) {
       if (attachModel == null)
         throw new IllegalArgumentException("attachModel must not be null!");
-      
+
       artifactMap.put(ATTACH_MODEL_ENTRY_NAME, attachModel);
     }
     else {
       throw new IllegalStateException("Unknown ParserType '" + modelType + "'!");
     }
-    
+
     artifactMap.put(PARSER_TAGGER_MODEL_ENTRY_NAME, parserTagger);
-    
+
     artifactMap.put(CHUNKER_TAGGER_MODEL_ENTRY_NAME, chunkerTagger);
-    
+
     artifactMap.put(HEAD_RULES_MODEL_ENTRY_NAME, headRules);
     checkArtifactMap();
   }
 
-  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel, 
+  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel,
       MaxentModel attachModel, POSModel parserTagger,
       ChunkerModel chunkerTagger, opennlp.tools.parser.HeadRules headRules,
       ParserType modelType) {
     this (languageCode, buildModel, checkModel, attachModel, parserTagger,
         chunkerTagger, headRules, modelType, null);
   }
-  
-  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel, 
-      POSModel parserTagger, ChunkerModel chunkerTagger, 
+
+  public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel,
+      POSModel parserTagger, ChunkerModel chunkerTagger,
       opennlp.tools.parser.HeadRules headRules, ParserType type,
       Map<String, String> manifestInfoEntries) {
-    this (languageCode, buildModel, checkModel, null, parserTagger, 
+    this (languageCode, buildModel, checkModel, null, parserTagger,
         chunkerTagger, headRules, type, manifestInfoEntries);
   }
-  
+
   public ParserModel(InputStream in) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, in);
   }
-  
+
   public ParserModel(File modelFile) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelFile);
   }
-  
+
   public ParserModel(URL modelURL) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelURL);
   }
-  
+
   @Override
   protected void createArtifactSerializers(
       Map<String, ArtifactSerializer> serializers) {
 
     super.createArtifactSerializers(serializers);
-    
+
     // In 1.6.x the headrules artifact is serialized with the new API
     // which uses the Serializeable interface
     // This change is not backward compatible with the 1.5.x models.
     // In order to laod 1.5.x model the English headrules serializer must be
     // put on the serializer map.
-    
+
     if (getVersion().getMajor() == 1 && getVersion().getMinor() == 5) {
     	serializers.put("headrules", new HeadRulesSerializer());
     }
-    
+
     serializers.put("postagger", new POSModelSerializer());
     serializers.put("chunker", new ChunkerModelSerializer());
   }
-  
+
   public ParserType getParserType () {
     return ParserType.parse(getManifestProperty(PARSER_TYPE));
   }
-  
+
   public MaxentModel getBuildModel() {
     return (MaxentModel) artifactMap.get(BUILD_MODEL_ENTRY_NAME);
   }
@@ -198,7 +198,7 @@ public class ParserModel extends BaseMod
   public MaxentModel getAttachModel() {
     return (MaxentModel) artifactMap.get(ATTACH_MODEL_ENTRY_NAME);
   }
-  
+
   public POSModel getParserTaggerModel() {
     return (POSModel) artifactMap.get(PARSER_TAGGER_MODEL_ENTRY_NAME);
   }
@@ -208,13 +208,13 @@ public class ParserModel extends BaseMod
   }
 
   public opennlp.tools.parser.HeadRules getHeadRules() {
-    return (opennlp.tools.parser.HeadRules) 
+    return (opennlp.tools.parser.HeadRules)
         artifactMap.get(HEAD_RULES_MODEL_ENTRY_NAME);
   }
 
   // TODO: Update model methods should make sure properties are copied correctly ...
   public ParserModel updateBuildModel(MaxentModel buildModel) {
-    return new ParserModel(getLanguage(), buildModel, getCheckModel(), getAttachModel(), 
+    return new ParserModel(getLanguage(), buildModel, getCheckModel(), getAttachModel(),
         getParserTaggerModel(), getParserChunkerModel(),
         getHeadRules(), getParserType());
   }
@@ -224,7 +224,7 @@ public class ParserModel extends BaseMod
         getAttachModel(), getParserTaggerModel(),
         getParserChunkerModel(), getHeadRules(), getParserType());
   }
-  
+
   public ParserModel updateTaggerModel(POSModel taggerModel) {
     return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(),
         taggerModel, getParserChunkerModel(), getHeadRules(), getParserType());
@@ -234,17 +234,17 @@ public class ParserModel extends BaseMod
     return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(),
         getParserTaggerModel(), chunkModel, getHeadRules(), getParserType());
   }
-  
+
   @Override
   protected void validateArtifactMap() throws InvalidFormatException {
     super.validateArtifactMap();
-    
+
     if (!(artifactMap.get(BUILD_MODEL_ENTRY_NAME)  instanceof AbstractModel)) {
       throw new InvalidFormatException("Missing the build model!");
     }
-    
+
     ParserType modelType = getParserType();
-    
+
     if (modelType != null) {
       if (ParserType.CHUNKING.equals(modelType)) {
         if (artifactMap.get(ATTACH_MODEL_ENTRY_NAME) != null)
@@ -261,19 +261,19 @@ public class ParserModel extends BaseMod
     else {
       throw new InvalidFormatException("Missing the parser type property!");
     }
-    
+
     if (!(artifactMap.get(CHECK_MODEL_ENTRY_NAME)  instanceof AbstractModel)) {
       throw new InvalidFormatException("Missing the check model!");
     }
-    
+
     if (!(artifactMap.get(PARSER_TAGGER_MODEL_ENTRY_NAME)  instanceof POSModel)) {
       throw new InvalidFormatException("Missing the tagger model!");
     }
-    
+
     if (!(artifactMap.get(CHUNKER_TAGGER_MODEL_ENTRY_NAME)  instanceof ChunkerModel)) {
       throw new InvalidFormatException("Missing the chunker model!");
     }
-    
+
     if (!(artifactMap.get(HEAD_RULES_MODEL_ENTRY_NAME)  instanceof HeadRules)) {
       throw new InvalidFormatException("Missing the head rules!");
     }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserType.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserType.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserType.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserType.java Fri May  2 12:34:23 2014
@@ -20,7 +20,7 @@ package opennlp.tools.parser;
 public enum ParserType {
   CHUNKING,
   TREEINSERT;
-  
+
   public static ParserType parse(String type) {
     if (ParserType.CHUNKING.name().equals(type)) {
       return ParserType.CHUNKING;

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/PosSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/PosSampleStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/PosSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/PosSampleStream.java Fri May  2 12:34:23 2014
@@ -30,22 +30,22 @@ public class PosSampleStream extends Fil
   }
 
   public POSSample read() throws IOException {
-    
+
     Parse parse = samples.read();
-    
+
     if (parse != null) {
-      
+
       Parse[] nodes = parse.getTagNodes();
-      
+
       String toks[] = new String[nodes.length];
       String preds[] = new String[nodes.length];
-      
+
       for (int ti=0; ti < nodes.length; ti++) {
         Parse tok = nodes[ti];
         toks[ti] = tok.getCoveredText();
         preds[ti] = tok.getType();
       }
-      
+
       return new POSSample(toks, preds);
     }
     else {

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java Fri May  2 12:34:23 2014
@@ -81,7 +81,7 @@ public class Parser extends AbstractBott
             new ChunkContextGenerator(ChunkerME.DEFAULT_BEAM_SIZE)),
             model.getHeadRules(), beamSize, advancePercentage);
   }
-  
+
   public Parser(ParserModel model) {
     this(model, defaultBeamSize, defaultAdvancePercentage);
   }
@@ -258,47 +258,47 @@ public class Parser extends AbstractBott
     return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
   }
 
-  public static void mergeReportIntoManifest(Map<String, String> manifest, 
+  public static void mergeReportIntoManifest(Map<String, String> manifest,
       Map<String, String> report, String namespace) {
-    
+
     for (Map.Entry<String, String> entry : report.entrySet()) {
       manifest.put(namespace + "." + entry.getKey(), entry.getValue());
     }
   }
-  
+
   public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams)
           throws IOException {
-    
+
     System.err.println("Building dictionary");
-    
+
     Dictionary mdict = buildDictionary(parseSamples, rules, mlParams);
-    
+
     parseSamples.reset();
-    
+
     Map<String, String> manifestInfoEntries = new HashMap<String, String>();
-    
+
     // build
     System.err.println("Training builder");
     ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict);
     Map<String, String> buildReportMap = new HashMap<String, String>();
     MaxentModel buildModel = TrainUtil.train(bes, mlParams.getSettings("build"), buildReportMap);
     mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build");
-    
+
     parseSamples.reset();
-    
+
     // tag
-    POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), 
+    POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples),
         mlParams.getParameters("tagger"), null, null);
-    
+
     parseSamples.reset();
-    
+
     // chunk
-    ChunkerModel chunkModel = ChunkerME.train(languageCode, 
+    ChunkerModel chunkModel = ChunkerME.train(languageCode,
         new ChunkSampleStream(parseSamples),
         new ChunkContextGenerator(), mlParams.getParameters("chunker"));
-    
+
     parseSamples.reset();
-    
+
     // check
     System.err.println("Training checker");
     ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK);
@@ -319,7 +319,7 @@ public class Parser extends AbstractBott
   @Deprecated
   public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut)
       throws IOException {
-    
+
     TrainingParameters params = new TrainingParameters();
     params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
 

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java Fri May  2 12:34:23 2014
@@ -60,7 +60,7 @@ public class HeadRules implements opennl
       artifact.serialize(new OutputStreamWriter(out, "UTF-8"));
     }
   }
-  
+
   private static class HeadRule {
     public boolean leftToRight;
     public String[] tags;
@@ -74,7 +74,7 @@ public class HeadRules implements opennl
 
       this.tags = tags;
     }
-    
+
     @Override
     public boolean equals(Object obj) {
       if (obj == this) {
@@ -82,8 +82,8 @@ public class HeadRules implements opennl
       }
       else if (obj instanceof HeadRule) {
         HeadRule rule = (HeadRule) obj;
-        
-        return (rule.leftToRight == leftToRight) && 
+
+        return (rule.leftToRight == leftToRight) &&
             Arrays.equals(rule.tags, tags);
       }
       else {
@@ -241,10 +241,10 @@ public class HeadRules implements opennl
    * Writes the head rules to the writer in a format suitable for loading
    * the head rules again with the constructor. The encoding must be
    * taken into account while working with the writer and reader.
-   * <p> 
+   * <p>
    * After the entries have been written, the writer is flushed.
    * The writer remains open after this method returns.
-   * 
+   *
    * @param writer
    * @throws IOException
    */
@@ -276,10 +276,10 @@ public class HeadRules implements opennl
 
       writer.write('\n');
     }
-    
+
     writer.flush();
   }
-  
+
   @Override
   public boolean equals(Object obj) {
     if (obj == this) {
@@ -287,7 +287,7 @@ public class HeadRules implements opennl
     }
     else if (obj instanceof HeadRules) {
       HeadRules rules = (HeadRules) obj;
-      
+
       return rules.headRules.equals(headRules) &&
           rules.punctSet.equals(punctSet);
     }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java Fri May  2 12:34:23 2014
@@ -45,16 +45,16 @@ import opennlp.tools.util.model.Serializ
 /**
  * Class for storing the Ancora Spanish head rules associated with parsing. The headrules
  * are specified in $src/main/resources/es-head-rules
- *  
+ *
  * NOTE: This class has been adapted from opennlp.tools.parser.lang.en.HeadRules
- * 
+ *
  * The main change is the constituents search direction in the first for loop.
- * 
+ *
  * Note also the change in the return of the getHead() method: In Apache OpenNLP
  * lang.en.HeadRules class: return constituents[ci].getHead(); Now: return constituents[ci];
- * 
- * Other changes include removal of deprecated methods we do not need to use. 
- * 
+ *
+ * Other changes include removal of deprecated methods we do not need to use.
+ *
  */
 public class AncoraSpanishHeadRules implements opennlp.tools.parser.HeadRules, GapLabeler, SerializableArtifact {
 
@@ -83,7 +83,7 @@ public class AncoraSpanishHeadRules impl
 
       this.tags = tags;
     }
-    
+
     @Override
     public boolean equals(Object obj) {
       if (obj == this) {
@@ -91,8 +91,8 @@ public class AncoraSpanishHeadRules impl
       }
       else if (obj instanceof HeadRule) {
         HeadRule rule = (HeadRule) obj;
-        
-        return (rule.leftToRight == leftToRight) && 
+
+        return (rule.leftToRight == leftToRight) &&
             Arrays.equals(rule.tags, tags);
       }
       else {
@@ -104,7 +104,7 @@ public class AncoraSpanishHeadRules impl
   private Map<String, HeadRule> headRules;
   private Set<String> punctSet;
 
-  
+
 
   /**
    * Creates a new set of head rules based on the specified reader.
@@ -136,7 +136,7 @@ public class AncoraSpanishHeadRules impl
     HeadRule hr;
       if (type.equals("SN") || type.equals("GRUP.NOM")) {
       String[] tags1 = {"AQA.*","AQC.*","GRUP\\.A","S\\.A","NC.*S.*", "NP.*","NC.*P.*", "GRUP\\.NOM"};
-      
+
       for (int i = 0; i < constituents.length; i++) {
         for (int t = tags1.length - 1; t >= 0; t--) {
           if (constituents[i].getType().matches(tags1[t])) {
@@ -241,10 +241,10 @@ public class AncoraSpanishHeadRules impl
    * Writes the head rules to the writer in a format suitable for loading
    * the head rules again with the constructor. The encoding must be
    * taken into account while working with the writer and reader.
-   * <p> 
+   * <p>
    * After the entries have been written, the writer is flushed.
    * The writer remains open after this method returns.
-   * 
+   *
    * @param writer
    * @throws IOException
    */
@@ -276,10 +276,10 @@ public class AncoraSpanishHeadRules impl
 
       writer.write('\n');
     }
-    
+
     writer.flush();
   }
-  
+
   @Override
   public boolean equals(Object obj) {
     if (obj == this) {
@@ -287,7 +287,7 @@ public class AncoraSpanishHeadRules impl
     }
     else if (obj instanceof AncoraSpanishHeadRules) {
       AncoraSpanishHeadRules rules = (AncoraSpanishHeadRules) obj;
-      
+
       return rules.headRules.equals(headRules) &&
           rules.punctSet.equals(punctSet);
     }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java Fri May  2 12:34:23 2014
@@ -80,7 +80,7 @@ public class BuildContextGenerator exten
     if (p1 != null) {
       punct2s=p1.getNextPunctuationSet();
     }
-    
+
 
     List<Parse> rf;
     if (index == 0) {
@@ -98,7 +98,7 @@ public class BuildContextGenerator exten
     if (p_1 != null) {
       punct_2s = p_1.getPreviousPunctuationSet();
     }
-    
+
     String consp_2 = cons(p_2, -2);
     String consp_1 = cons(p_1, -1);
     String consp0 = cons(p0, 0);

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java Fri May  2 12:34:23 2014
@@ -99,8 +99,8 @@ public class Parser extends AbstractBott
   private int[] attachments;
 
   public Parser(ParserModel model, int beamSize, double advancePercentage) {
-    this(model.getBuildModel(), model.getAttachModel(), model.getCheckModel(), 
-        new POSTaggerME(model.getParserTaggerModel()), 
+    this(model.getBuildModel(), model.getAttachModel(), model.getCheckModel(),
+        new POSTaggerME(model.getParserTaggerModel()),
         new ChunkerME(model.getParserChunkerModel(),
         ChunkerME.DEFAULT_BEAM_SIZE,
         new ParserChunkerSequenceValidator(model.getParserChunkerModel()),
@@ -108,11 +108,11 @@ public class Parser extends AbstractBott
         model.getHeadRules(),
         beamSize, advancePercentage);
   }
-  
+
   public Parser(ParserModel model) {
     this(model, defaultBeamSize, defaultAdvancePercentage);
   }
-  
+
   private Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize, double advancePercentage) {
     super(tagger,chunker,headRules,beamSize,advancePercentage);
     this.buildModel = buildModel;
@@ -435,26 +435,26 @@ public class Parser extends AbstractBott
   public static ParserModel train(String languageCode,
       ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams)
   throws IOException {
-    
+
     Map<String, String> manifestInfoEntries = new HashMap<String, String>();
-    
+
     System.err.println("Building dictionary");
     Dictionary mdict = buildDictionary(parseSamples, rules, mlParams);
-    
+
     parseSamples.reset();
-    
+
     // tag
     POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(
         parseSamples), mlParams.getParameters("tagger"), null, null);
-    
+
     parseSamples.reset();
-    
+
     // chunk
     ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(
         parseSamples), new ChunkContextGenerator(), mlParams.getParameters("chunker"));
-    
+
     parseSamples.reset();
-    
+
     // build
     System.err.println("Training builder");
     ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules,
@@ -462,9 +462,9 @@ public class Parser extends AbstractBott
     Map<String, String> buildReportMap = new HashMap<String, String>();
     MaxentModel buildModel = TrainUtil.train(bes, mlParams.getSettings("build"), buildReportMap);
     opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build");
-    
+
     parseSamples.reset();
-    
+
     // check
     System.err.println("Training checker");
     ObjectStream<Event>  kes = new ParserEventStream(parseSamples, rules,
@@ -472,27 +472,27 @@ public class Parser extends AbstractBott
     Map<String, String> checkReportMap = new HashMap<String, String>();
     MaxentModel checkModel = TrainUtil.train(kes, mlParams.getSettings("check"), checkReportMap);
     opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check");
-    
+
     parseSamples.reset();
-    
-    // attach 
+
+    // attach
     System.err.println("Training attacher");
     ObjectStream<Event>  attachEvents = new ParserEventStream(parseSamples, rules,
         ParserEventTypeEnum.ATTACH);
     Map<String, String> attachReportMap = new HashMap<String, String>();
     MaxentModel attachModel = TrainUtil.train(attachEvents, mlParams.getSettings("attach"), attachReportMap);
     opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest(manifestInfoEntries, attachReportMap, "attach");
-    
+
     // TODO: Remove cast for HeadRules
     return new ParserModel(languageCode, buildModel, checkModel,
-        attachModel, posModel, chunkModel, 
+        attachModel, posModel, chunkModel,
         (opennlp.tools.parser.lang.en.HeadRules) rules, ParserType.TREEINSERT, manifestInfoEntries);
   }
-  
+
   public static ParserModel train(String languageCode,
       ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut)
       throws IOException {
-    
+
     TrainingParameters params = new TrainingParameters();
     params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
 
@@ -504,10 +504,10 @@ public class Parser extends AbstractBott
     params.put("check", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
     params.put("build", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
     params.put("build", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
-    
+
     return train(languageCode, parseSamples, rules, params);
   }
-  
+
   @Deprecated
   public static AbstractModel train(ObjectStream<Event>  es, int iterations, int cut) throws java.io.IOException {
     return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java Fri May  2 12:34:23 2014
@@ -27,12 +27,12 @@ public interface MutableTagDictionary ex
    * Associates the specified tags with the specified word. If the dictionary
    * previously contained keys for the word, the old tags are replaced by the
    * specified tags.
-   * 
+   *
    * @param word
    *          word with which the specified tags is to be associated
    * @param tags
    *          tags to be associated with the specified word
-   * 
+   *
    * @return the previous tags associated with the word, or null if there was no
    *         mapping for word.
    */
@@ -40,7 +40,7 @@ public interface MutableTagDictionary ex
 
   /**
    * Whether if the dictionary is case sensitive or not
-   * 
+   *
    * @return true if the dictionary is case sensitive
    */
   // TODO: move to TagDictionary, can't do it now because of backward

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java Fri May  2 12:34:23 2014
@@ -47,7 +47,7 @@ public class POSDictionary implements It
   private Map<String, String[]> dictionary;
 
   private boolean caseSensitive = true;
-  
+
   /**
    * Initializes an empty case sensitive {@link POSDictionary}.
    */
@@ -70,7 +70,7 @@ public class POSDictionary implements It
    * @param file The file name for the tag dictionary.
    *
    * @throws IOException when the specified file can not be read.
-   * 
+   *
    * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
    */
   @Deprecated
@@ -86,7 +86,7 @@ public class POSDictionary implements It
    * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
    *
    * @throws IOException when the specified file can not be read.
-   * 
+   *
    * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
    */
   @Deprecated
@@ -103,7 +103,7 @@ public class POSDictionary implements It
    * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
    *
    * @throws IOException when the specified file can not be read.
-   * 
+   *
    * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
    */
   @Deprecated
@@ -118,7 +118,7 @@ public class POSDictionary implements It
    * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
    *
    * @throws IOException when the specified file can not be read.
-   * 
+   *
    * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
    */
   @Deprecated
@@ -161,12 +161,12 @@ public class POSDictionary implements It
    * Associates the specified tags with the specified word. If the dictionary
    * previously contained the word, the old tags are replaced by the specified
    * ones.
-   * 
+   *
    * @param word
    *          The word to be added to the dictionary.
    * @param tags
    *          The set of tags associated with the specified word.
-   * 
+   *
    * @deprecated Use {@link #put(String, String[])} instead
    */
   void addTags(String word, String... tags) {
@@ -306,18 +306,18 @@ public class POSDictionary implements It
       }});
 
     newPosDict.caseSensitive = isCaseSensitive;
-    
+
     // TODO: The dictionary API needs to be improved to do this better!
     if (!isCaseSensitive) {
       Map<String, String[]> lowerCasedDictionary = new HashMap<String, String[]>();
-      
+
       for (Map.Entry<String, String[]> entry : newPosDict.dictionary.entrySet()) {
         lowerCasedDictionary.put(StringUtil.toLowerCase(entry.getKey()), entry.getValue());
       }
-      
+
       newPosDict.dictionary = lowerCasedDictionary;
     }
-    
+
     return newPosDict;
   }
 

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java Fri May  2 12:34:23 2014
@@ -31,7 +31,7 @@ public class POSEvaluator extends Evalua
   private POSTagger tagger;
 
   private Mean wordAccuracy = new Mean();
-  
+
   /**
    * Initializes the current instance.
    *
@@ -42,7 +42,7 @@ public class POSEvaluator extends Evalua
     super(listeners);
     this.tagger = tagger;
   }
-  
+
   /**
    * Evaluates the given reference {@link POSSample} object.
    *
@@ -51,15 +51,15 @@ public class POSEvaluator extends Evalua
    * tags are then used to update the word accuracy score.
    *
    * @param reference the reference {@link POSSample}.
-   * 
+   *
    * @return the predicted {@link POSSample}.
    */
   @Override
   protected POSSample processSample(POSSample reference) {
-    
+
     String predictedTags[] = tagger.tag(reference.getSentence(), reference.getAddictionalContext());
     String referenceTags[] = reference.getTags();
-    
+
     for (int i = 0; i < referenceTags.length; i++) {
       if (referenceTags[i].equals(predictedTags[i])) {
         wordAccuracy.add(1);
@@ -68,7 +68,7 @@ public class POSEvaluator extends Evalua
         wordAccuracy.add(0);
       }
     }
-    
+
     return new POSSample(reference.getSentence(), predictedTags);
   }
 
@@ -87,13 +87,13 @@ public class POSEvaluator extends Evalua
   /**
    * Retrieves the total number of words considered
    * in the evaluation.
-   * 
+   *
    * @return the word count
    */
   public long getWordCount() {
     return wordAccuracy.count();
   }
-  
+
   /**
    * Represents this objects as human readable {@link String}.
    */

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java Fri May  2 12:34:23 2014
@@ -44,7 +44,7 @@ import opennlp.tools.util.model.BaseMode
 public final class POSModel extends BaseModel {
 
   private static final String COMPONENT_NAME = "POSTaggerME";
-  
+
   public static final String POS_MODEL_ENTRY_NAME = "pos.model";
 
   /**
@@ -79,7 +79,7 @@ public final class POSModel extends Base
         throw new IllegalArgumentException("The maxentPosModel param must not be null!");
 
     artifactMap.put(POS_MODEL_ENTRY_NAME, posModel);
-    // TODO: This fails probably for the sequence model ... ?! 
+    // TODO: This fails probably for the sequence model ... ?!
     // checkArtifactMap();
   }
 
@@ -87,7 +87,7 @@ public final class POSModel extends Base
       Map<String, String> manifestInfoEntries, POSTaggerFactory posFactory) {
     this(languageCode, posModel, POSTaggerME.DEFAULT_BEAM_SIZE, manifestInfoEntries, posFactory);
   }
-  
+
   public POSModel(String languageCode, MaxentModel posModel, int beamSize,
       Map<String, String> manifestInfoEntries, POSTaggerFactory posFactory) {
 
@@ -99,15 +99,15 @@ public final class POSModel extends Base
     artifactMap.put(POS_MODEL_ENTRY_NAME, posModel);
     checkArtifactMap();
   }
-  
+
   public POSModel(InputStream in) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, in);
   }
-  
+
   public POSModel(File modelFile) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelFile);
   }
-  
+
   public POSModel(URL modelURL) throws IOException, InvalidFormatException {
     super(COMPONENT_NAME, modelURL);
   }
@@ -149,17 +149,17 @@ public final class POSModel extends Base
   }
 
   public SequenceClassificationModel<String> getPosSequenceModel() {
-    
+
     Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
-    
+
     if (artifactMap.get(POS_MODEL_ENTRY_NAME) instanceof MaxentModel) {
       String beamSizeString = manifest.getProperty(BeamSearch.BEAM_SIZE_PARAMETER);
-      
+
       int beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
       if (beamSizeString != null) {
         beamSize = Integer.parseInt(beamSizeString);
       }
-      
+
       return new BeamSearch<>(beamSize, (MaxentModel) artifactMap.get(POS_MODEL_ENTRY_NAME));
     }
     else if (artifactMap.get(POS_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
@@ -169,17 +169,17 @@ public final class POSModel extends Base
       return null;
     }
   }
-  
+
   /**
    * Retrieves the tag dictionary.
-   * 
+   *
    * @return tag dictionary or null if not used
-   * 
+   *
    * @deprecated Use {@link POSModel#getFactory()} to get a
    *             {@link POSTaggerFactory} and
    *             {@link POSTaggerFactory#getTagDictionary()} to get a
    *             {@link TagDictionary}.
-   * 
+   *
    * @throws IllegalStateException
    *           if the TagDictionary is not an instance of POSDictionary
    */
@@ -200,7 +200,7 @@ public final class POSModel extends Base
     }
     return null;
   }
-  
+
   public POSTaggerFactory getFactory() {
     return (POSTaggerFactory) this.toolFactory;
   }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSample.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSample.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSample.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSample.java Fri May  2 12:34:23 2014
@@ -39,7 +39,7 @@ public class POSSample {
   public POSSample(String sentence[], String tags[]) {
     this(sentence, tags, null);
   }
-  
+
   public POSSample(List<String> sentence, List<String> tags) {
     this(sentence, tags, null);
   }
@@ -64,12 +64,12 @@ public class POSSample {
     }
     this.additionalContext = ac;
   }
-  
+
   public POSSample(String sentence[], String tags[],
       String[][] additionalContext) {
     this(Arrays.asList(sentence), Arrays.asList(tags), additionalContext);
   }
-  
+
   private void checkArguments() {
     if (sentence.size() != tags.size()) {
       throw new IllegalArgumentException(
@@ -84,7 +84,7 @@ public class POSSample {
         throw new IllegalArgumentException("null elements are not allowed in tags!");
       }
   }
-  
+
   public String[] getSentence() {
     return sentence.toArray(new String[sentence.size()]);
   }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSampleEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSampleEventStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSampleEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSSampleEventStream.java Fri May  2 12:34:23 2014
@@ -69,7 +69,7 @@ public class POSSampleEventStream extend
     List<Event> events = generateEvents(sentence, tags, ac, cg);
     return events.iterator();
   }
-  
+
   public static List<Event> generateEvents(String[] sentence, String[] tags,
       Object[] additionalContext, POSContextGenerator cg) {
     List<Event> events = new ArrayList<Event>(sentence.length);