You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/23 22:17:56 UTC

[39/50] [abbrv] incubator-joshua git commit: Merge branch 'master' into 7

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
index 9b07a04,0000000..a2975c5
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
@@@ -1,730 -1,0 +1,735 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import static org.apache.joshua.util.Constants.TM_PREFIX;
 +import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
 +import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
 +
 +import java.io.BufferedReader;
 +import java.io.File;
 +import java.io.FileReader;
 +import java.io.FileWriter;
 +import java.io.IOException;
 +import java.io.PrintWriter;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +
 +import org.apache.joshua.decoder.ff.StatefulFF;
 +import org.apache.joshua.decoder.ff.fragmentlm.Tree;
 +import org.apache.joshua.util.FormatUtils;
 +import org.apache.joshua.util.Regex;
 +import org.apache.joshua.util.io.LineReader;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.LogManager;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +/**
 + * Configuration file for Joshua decoder.
 + *
 + * When adding new features to Joshua, any new configurable parameters should be added to this
 + * class.
 + *
 + * @author Zhifei Li, zhifei.work@gmail.com
 + * @author Matt Post post@cs.jhu.edu
 + */
 +public class JoshuaConfiguration {
 +
 +  private static final Logger LOG = LoggerFactory.getLogger(JoshuaConfiguration.class);
 +
 +  // whether to construct a StructuredTranslation object for each request instead of
 +  // printing to stdout. Used when the Decoder is used from Java directly.
 +  public Boolean use_structured_output = false;
 +
 +  // If set to true, Joshua will lowercase the input, creating an annotation that marks the
 +  // original case
 +  public boolean lowercase = false;
 +
 +  // If set to true, Joshua will recapitalize the output by projecting the case from aligned
 +  // source-side words
 +  public boolean project_case = false;
 +
 +  // List of grammar files to read
-   public ArrayList<String> tms = new ArrayList<String>();
++  public ArrayList<String> tms = new ArrayList<>();
 +
 +  // A rule cache for commonly used tries to avoid excess object allocations
 +  // Testing shows there's up to ~95% hit rate when cache size is 5000 Trie nodes.
-   public Integer cachedRuleSize = new Integer(5000);
++  public Integer cachedRuleSize = 5000;
 +
 +  /*
 +   * The file to read the weights from (part of the sparse features implementation). Weights can
 +   * also just be listed in the main config file.
 +   */
 +  public String weights_file = "";
 +  // Default symbols. The symbol here should be enclosed in square brackets.
 +  public String default_non_terminal = FormatUtils.ensureNonTerminalBrackets("X");
 +  public String goal_symbol = FormatUtils.ensureNonTerminalBrackets("GOAL");
 +
 +  /*
 +   * A list of OOV symbols in the form
 +   *
 +   * [X1] weight [X2] weight [X3] weight ...
 +   *
 +   * where the [X] symbols are nonterminals and the weights are weights. For each OOV word w in the
 +   * input sentence, Joshua will create rules of the form
 +   *
 +   * X1 -> w (weight)
 +   *
 +   * If this is empty, an unweighted default_non_terminal is used.
 +   */
 +  public class OOVItem implements Comparable<OOVItem> {
-     public String label;
++    public final String label;
 +
-     public float weight;
++    public final float weight;
 +
 +    OOVItem(String l, float w) {
 +      label = l;
 +      weight = w;
 +    }
 +    @Override
 +    public int compareTo(OOVItem other) {
 +      if (weight > other.weight)
 +        return -1;
 +      else if (weight < other.weight)
 +        return 1;
 +      return 0;
 +    }
 +  }
 +
 +  public ArrayList<OOVItem> oovList = null;
 +
 +  /*
 +   * Whether to segment OOVs into a lattice
 +   */
 +  public boolean segment_oovs = false;
 +
 +  /*
 +   * Enable lattice decoding.
 +   */
 +  public boolean lattice_decoding = false;
 +
 +  /*
 +   * If false, sorting of the complete grammar is done at load time. If true, grammar tries are not
 +   * sorted till they are first accessed. Amortized sorting means you get your first translation
 +   * much, much quicker (good for debugging), but that per-sentence decoding is a bit slower.
 +   */
 +  public boolean amortized_sorting = true;
 +  // syntax-constrained decoding
 +  public boolean constrain_parse = false;
 +
 +  public boolean use_pos_labels = false;
 +
 +  // oov-specific
 +  public boolean true_oovs_only = false;
 +
 +  /* Dynamic sentence-level filtering. */
 +  public boolean filter_grammar = false;
 +
 +  /* The cube pruning pop limit. Set to 0 for exhaustive pruning. */
 +  public int pop_limit = 100;
 +
 +  /* Maximum sentence length. Sentences longer than this are truncated. */
 +  public int maxlen = 200;
 +
 +  /*
 +   * N-best configuration.
 +   */
 +  // Make sure output strings in the n-best list are unique.
 +  public boolean use_unique_nbest = true;
 +
 +  /* Include the phrasal alignments in the output (not word-level alignmetns at the moment). */
 +  public boolean include_align_index = false;
 +
 +  /* The number of hypotheses to output by default. */
 +  public int topN = 1;
 +
 +  /**
 +   * This string describes the format of each line of output from the decoder (i.e., the
 +   * translations). The string can include arbitrary text and also variables. The following
 +   * variables are available:
 +   *
 +   * <pre>
 +   * - %i the 0-indexed sentence number
 +   * - %e the source string %s the translated sentence
 +   * - %S the translated sentence with some basic capitalization and denormalization
 +   * - %t the synchronous derivation
 +   * - %f the list of feature values (as name=value pairs)
 +   * - %c the model cost
 +   * - %w the weight vector
 +   * - %a the alignments between source and target words (currently unimplemented)
 +   * - %d a verbose, many-line version of the derivation
 +   * </pre>
 +   */
 +  public String outputFormat = "%i ||| %s ||| %f ||| %c";
 +
 +  /* The number of decoding threads to use (-threads). */
 +  public int num_parallel_decoders = 1;
 +
 +  /*
 +   * When true, _OOV is appended to all words that are passed through (useful for something like
 +   * transliteration on the target side
 +   */
 +  public boolean mark_oovs = false;
 +
 +  /* Enables synchronous parsing. */
 +  public boolean parse = false; // perform synchronous parsing
 +
 +
 +  /* A list of the feature functions. */
-   public ArrayList<String> features = new ArrayList<String>();
++  public ArrayList<String> features = new ArrayList<>();
 +
 +  /* A list of weights found in the main config file (instead of in a separate weights file) */
-   public ArrayList<String> weights = new ArrayList<String>();
++  public ArrayList<String> weights = new ArrayList<>();
 +
 +  /* Determines whether to expect JSON input or plain lines */
-   public enum INPUT_TYPE { plain, json };
++  public enum INPUT_TYPE { plain, json }
++
 +  public INPUT_TYPE input_type = INPUT_TYPE.plain;
 +
 +  /* Type of server. Not sure we need to keep the regular TCP one around. */
-   public enum SERVER_TYPE { none, TCP, HTTP };
++  public enum SERVER_TYPE { none, TCP, HTTP }
++
 +  public SERVER_TYPE server_type = SERVER_TYPE.TCP;
 +
 +  /* If set, Joshua will start a (multi-threaded, per "threads") TCP/IP server on this port. */
 +  public int server_port = 0;
 +
 +  /*
 +   * Whether to do forest rescoring. If set to true, the references are expected on STDIN along with
 +   * the input sentences in the following format:
 +   * 
 +   * input sentence ||| ||| reference1 ||| reference2 ...
 +   * 
 +   * (The second field is reserved for the output sentence for alignment and forced decoding).
 +   */
 +
 +  public boolean rescoreForest = false;
 +  public float rescoreForestWeight = 10.0f;
 +
 +  /*
 +   * Location of fragment mapping file, which maps flattened SCFG rules to their internal
 +   * representation.
 +   */
 +  public String fragmentMapFile = null;
 +
 +  /*
 +   * Whether to use soft syntactic constraint decoding /fuzzy matching, which allows that any
 +   * nonterminal may be substituted for any other nonterminal (except for OOV and GOAL)
 +   */
 +  public boolean fuzzy_matching = false;
 +
 +  public static final String SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME = "fuzzy_matching";
 +
 +  /***
 +   * Phrase-based decoding parameters.
 +   */
 +  
 +  /* The search algorithm: currently either "cky" or "stack" */
 +  public String search_algorithm = "cky";
 +
 +  /* The distortion limit */
 +  public int reordering_limit = 8;
 +
 +  /* The number of target sides considered for each source side (after sorting by model weight) */
 +  public int num_translation_options = 20;
 +
 +  /* If true, decode using a dot chart (standard CKY+); if false, use the much more efficient
 +   * version of Sennrich (SSST 2014)
 +   */
 +  public boolean use_dot_chart = true;
 +
 +  /* Moses compatibility */
 +  public boolean moses = false;
 +
 +  /* If true, just print out the weights found in the config file, and exit. */
 +  public boolean show_weights_and_quit = false;
 +
 +  /* Read input from a file (Moses compatible flag) */
 +  public String input_file = null;
 +
 +  /* Write n-best output to this file */
 +  public String n_best_file = null;
 +
 +  /* Whether to look at source side for special annotations */
 +  public boolean source_annotations = false;
 +
 +  /* Weights overridden from the command line */
 +  public String weight_overwrite = "";
 +
 +  /**
 +   * This method resets the state of JoshuaConfiguration back to the state after initialization.
 +   * This is useful when for example making different calls to the decoder within the same java
 +   * program, which otherwise leads to potential errors due to inconsistent state as a result of
 +   * loading the configuration multiple times without resetting etc.
 +   *
 +   * This leads to the insight that in fact it may be an even better idea to refactor the code and
 +   * make JoshuaConfiguration an object that is is created and passed as an argument, rather than a
 +   * shared static object. This is just a suggestion for the next step.
 +   *
 +   */
 +  public void reset() {
 +    LOG.info("Resetting the JoshuaConfiguration to its defaults ...");
 +    LOG.info("\n\tResetting the StatefullFF global state index ...");
 +    LOG.info("\n\t...done");
 +    StatefulFF.resetGlobalStateIndex();
-     tms = new ArrayList<String>();
++    tms = new ArrayList<>();
 +    weights_file = "";
 +    default_non_terminal = "[X]";
-     oovList = new ArrayList<OOVItem>();
++    oovList = new ArrayList<>();
 +    oovList.add(new OOVItem(default_non_terminal, 1.0f));
 +    goal_symbol = "[GOAL]";
 +    amortized_sorting = true;
 +    constrain_parse = false;
 +    use_pos_labels = false;
 +    true_oovs_only = false;
 +    filter_grammar = false;
 +    pop_limit = 100;
 +    maxlen = 200;
 +    use_unique_nbest = false;
 +    include_align_index = false;
 +    topN = 1;
 +    outputFormat = "%i ||| %s ||| %f ||| %c";
 +    num_parallel_decoders = 1;
 +    mark_oovs = false;
 +    // oracleFile = null;
 +    parse = false; // perform synchronous parsing
-     features = new ArrayList<String>();
-     weights = new ArrayList<String>();
++    features = new ArrayList<>();
++    weights = new ArrayList<>();
 +    server_port = 0;
 +
 +    reordering_limit = 8;
 +    num_translation_options = 20;
 +    LOG.info("...done");
 +  }
 +
 +  // ===============================================================
 +  // Methods
 +  // ===============================================================
 +
 +  /**
 +   * To process command-line options, we write them to a file that looks like the config file, and
 +   * then call readConfigFile() on it. It would be more general to define a class that sits on a
 +   * stream and knows how to chop it up, but this was quicker to implement.
 +   * 
 +   * @param options string array of command line options
 +   */
 +  public void processCommandLineOptions(String[] options) {
 +    try {
 +      File tmpFile = File.createTempFile("options", null, null);
 +      PrintWriter out = new PrintWriter(new FileWriter(tmpFile));
 +
 +      for (int i = 0; i < options.length; i++) {
 +        String key = options[i].substring(1);
 +        if (i + 1 == options.length || options[i + 1].startsWith("-")) {
 +          // if this is the last item, or if the next item
 +          // is another flag, then this is a boolean flag
 +          out.println(key + " = true");
 +
 +        } else {
 +          out.print(key + " =");
 +          while (i + 1 < options.length && ! options[i + 1].startsWith("-")) {
 +            out.print(String.format(" %s", options[i + 1]));
 +            i++;
 +          }
 +          out.println();
 +        }
 +      }
 +      out.close();
 +      
 +//      LOG.info("Parameters overridden from the command line:");
 +      this.readConfigFile(tmpFile.getCanonicalPath());
 +
 +      tmpFile.delete();
 +
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  public void readConfigFile(String configFile) throws IOException {
 +
 +    LineReader configReader = new LineReader(configFile, false);
 +    try {
 +      for (String line : configReader) {
 +        line = line.trim(); // .toLowerCase();
 +
 +        if (Regex.commentOrEmptyLine.matches(line))
 +          continue;
 +
 +        /*
 +         * There are two kinds of substantive (non-comment, non-blank) lines: parameters and feature
 +         * values. Parameters match the pattern "key = value"; all other substantive lines are
 +         * interpreted as features.
 +         */
 +
-         if (line.indexOf("=") != -1) { // parameters; (not feature function)
++        if (line.contains("=")) { // parameters; (not feature function)
 +          String[] fds = Regex.equalsWithSpaces.split(line, 2);
 +          if (fds.length < 2) {
 +            LOG.warn("skipping config file line '{}'", line);
 +            continue;
 +          }
 +
 +          String parameter = normalize_key(fds[0]);
 +
 +          if (parameter.equals(normalize_key("lm"))) {
 +            /* This is deprecated. This support old LM lines of the form
 +             * 
 +             *   lm = berkeleylm 5 false false 100 lm.gz
 +             * 
 +             * LMs are now loaded as general feature functions, so we transform that to either
 +             * 
 +             *   LanguageModel -lm_order 5 -lm_type berkeleylm -lm_file lm.gz
 +             * 
 +             * If the line were state minimizing:
 +             * 
 +             *   lm = kenlm 5 true false 100 lm.gz
 +             *              
 +             * StateMinimizingLanguageModel -lm_order 5 -lm_file lm.gz
 +             */
 +
 +            String[] tokens = fds[1].split("\\s+");
 +            if (tokens[2].equals("true"))
 +              features.add(String.format("StateMinimizingLanguageModel -lm_type kenlm -lm_order %s -lm_file %s",
 +                  tokens[1], tokens[5]));
 +            else
 +              features.add(String.format("LanguageModel -lm_type %s -lm_order %s -lm_file %s",
 +                  tokens[0], tokens[1], tokens[5]));
 +
 +          } else if (parameter.equals(normalize_key(TM_PREFIX))) {
 +            /* If found, convert old format:
 +             *   tm = TYPE OWNER MAXSPAN PATH
 +             * to new format
 +             *   tm = TYPE -owner OWNER -maxspan MAXSPAN -path PATH    
 +             */
 +            String tmLine = fds[1];
 +
 +            String[] tokens = fds[1].split("\\s+");
 +            if (! tokens[1].startsWith("-")) { // old format
 +              tmLine = String.format("%s -owner %s -maxspan %s -path %s", tokens[0], tokens[1], tokens[2], tokens[3]);
 +              LOG.warn("Converting deprecated TM line from '{}' -> '{}'", fds[1], tmLine);
 +            }
 +            tms.add(tmLine);
 +
 +          } else if (parameter.equals("v")) {
 +
 +            // This is already handled in ArgsParser, skip it here, easier than removing it there
 +
 +          } else if (parameter.equals(normalize_key("parse"))) {
 +            parse = Boolean.parseBoolean(fds[1]);
 +            LOG.debug("parse: {}", parse);
 +
 +          } else if (parameter.equals(normalize_key("oov-list"))) {
 +            if (new File(fds[1]).exists()) {
-               oovList = new ArrayList<OOVItem>();
++              oovList = new ArrayList<>();
 +              try {
 +                File file = new File(fds[1]);
 +                BufferedReader br = new BufferedReader(new FileReader(file));
 +                try {
 +                  String str = br.readLine();
 +                  while (str != null) {
 +                    String[] tokens = str.trim().split("\\s+");
 +
 +                    oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[0]),
 +                            (float) Math.log(Float.parseFloat(tokens[1]))));
 +
 +                    str = br.readLine();
 +                  }
 +                  br.close();
 +                } catch(IOException e){
 +                  System.out.println(e);
 +                }
 +              } catch(IOException e){
 +                System.out.println(e);
 +              }
 +              Collections.sort(oovList);
 +
 +            } else {
 +              String[] tokens = fds[1].trim().split("\\s+");
 +              if (tokens.length % 2 != 0) {
 +                throw new RuntimeException(String.format("* FATAL: invalid format for '%s'", fds[0]));
 +              }
-               oovList = new ArrayList<OOVItem>();
++              oovList = new ArrayList<>();
 +
 +              for (int i = 0; i < tokens.length; i += 2)
 +                oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[i]),
 +                    (float) Math.log(Float.parseFloat(tokens[i + 1]))));
 +
 +              Collections.sort(oovList);
 +            }
 +
 +          } else if (parameter.equals(normalize_key("lattice-decoding"))) {
 +            lattice_decoding = true;
 +
 +          } else if (parameter.equals(normalize_key("segment-oovs"))) {
 +            segment_oovs = true;
 +            lattice_decoding = true;
 +
 +          } else if (parameter.equals(normalize_key("default-non-terminal"))) {
 +            default_non_terminal = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
 +            LOG.debug("default_non_terminal: {}", default_non_terminal);
 +
 +          } else if (parameter.equals(normalize_key("goal-symbol"))) {
 +            goal_symbol = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
 +            LOG.debug("goalSymbol: {}", goal_symbol);
 +
 +          } else if (parameter.equals(normalize_key("weights-file"))) {
 +            weights_file = fds[1];
 +
 +          } else if (parameter.equals(normalize_key("constrain_parse"))) {
 +            constrain_parse = Boolean.parseBoolean(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("true_oovs_only"))) {
 +            true_oovs_only = Boolean.parseBoolean(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("filter-grammar"))) {
 +            filter_grammar = Boolean.parseBoolean(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("amortize"))) {
 +            amortized_sorting = Boolean.parseBoolean(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("use_pos_labels"))) {
 +            use_pos_labels = Boolean.parseBoolean(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("use_unique_nbest"))) {
 +            use_unique_nbest = Boolean.valueOf(fds[1]);
 +            LOG.debug("use_unique_nbest: {}", use_unique_nbest);
 +
 +          } else if (parameter.equals(normalize_key("output-format"))) {
 +            outputFormat = fds[1];
 +            LOG.debug("output-format: {}", outputFormat);
 +
 +          } else if (parameter.equals(normalize_key("include_align_index"))) {
 +            include_align_index = Boolean.valueOf(fds[1]);
 +            LOG.debug("include_align_index: {}", include_align_index);
 +
 +          } else if (parameter.equals(normalize_key("top_n"))) {
 +            topN = Integer.parseInt(fds[1]);
 +            LOG.debug("topN: {}", topN);
 +
 +          } else if (parameter.equals(normalize_key("num_parallel_decoders"))
 +              || parameter.equals(normalize_key("threads"))) {
 +            num_parallel_decoders = Integer.parseInt(fds[1]);
 +            if (num_parallel_decoders <= 0) {
 +              throw new IllegalArgumentException(
 +                  "Must specify a positive number for num_parallel_decoders");
 +            }
 +            LOG.debug("num_parallel_decoders: {}", num_parallel_decoders);
 +
 +          } else if (parameter.equals(normalize_key("mark_oovs"))) {
 +            mark_oovs = Boolean.valueOf(fds[1]);
 +            LOG.debug("mark_oovs: {}", mark_oovs);
 +
 +          } else if (parameter.equals(normalize_key("pop-limit"))) {
 +            pop_limit = Integer.parseInt(fds[1]);
 +            LOG.info("pop-limit: {}", pop_limit);
 +
 +          } else if (parameter.equals(normalize_key("input-type"))) {
-             if (fds[1].equals("json")) {
++            switch (fds[1]) {
++            case "json":
 +              input_type = INPUT_TYPE.json;
-             } else if (fds[1].equals("plain")) {
++              break;
++            case "plain":
 +              input_type = INPUT_TYPE.plain;
-             } else {
-               throw new RuntimeException(String.format("* FATAL: invalid server type '%s'", fds[1]));
++              break;
++            default:
++              throw new RuntimeException(
++                  String.format("* FATAL: invalid server type '%s'", fds[1]));
 +            }
 +            LOG.info("    input-type: {}", input_type);
 +
 +          } else if (parameter.equals(normalize_key("server-type"))) {
 +            if (fds[1].toLowerCase().equals("tcp"))
 +              server_type = SERVER_TYPE.TCP;
 +            else if (fds[1].toLowerCase().equals("http"))
 +              server_type = SERVER_TYPE.HTTP;
 +
 +            LOG.info("    server-type: {}", server_type);
 +
 +          } else if (parameter.equals(normalize_key("server-port"))) {
 +            server_port = Integer.parseInt(fds[1]);
 +            LOG.info("    server-port: {}", server_port);
 +
 +          } else if (parameter.equals(normalize_key("rescore-forest"))) {
 +            rescoreForest = true;
 +            LOG.info("    rescore-forest: {}", rescoreForest);
 +
 +          } else if (parameter.equals(normalize_key("rescore-forest-weight"))) {
 +            rescoreForestWeight = Float.parseFloat(fds[1]);
 +            LOG.info("    rescore-forest-weight: {}", rescoreForestWeight);
 +
 +          } else if (parameter.equals(normalize_key("maxlen"))) {
 +            // reset the maximum length
 +            maxlen = Integer.parseInt(fds[1]);
 +
 +          } else if (parameter.equals("c") || parameter.equals("config")) {
 +            // this was used to send in the config file, just ignore it
-             ;
 +
 +          } else if (parameter.equals(normalize_key("feature-function"))) {
 +            // add the feature to the list of features for later processing
 +            features.add(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("maxlen"))) {
 +            // add the feature to the list of features for later processing
 +            maxlen = Integer.parseInt(fds[1]);
 +
 +          } else if (parameter
 +              .equals(normalize_key(SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME))) {
 +            fuzzy_matching = Boolean.parseBoolean(fds[1]);
 +            LOG.debug("fuzzy_matching: {}", fuzzy_matching);
 +
 +          } else if (parameter.equals(normalize_key("fragment-map"))) {
 +            fragmentMapFile = fds[1];
 +            Tree.readMapping(fragmentMapFile);
 +
 +            /** PHRASE-BASED PARAMETERS **/
 +          } else if (parameter.equals(normalize_key("search"))) {
 +            search_algorithm = fds[1];
 +
 +            if (!search_algorithm.equals("cky") && !search_algorithm.equals("stack")) {
 +              throw new RuntimeException(
 +                  "-search must be one of 'stack' (for phrase-based decoding) " +
 +                      "or 'cky' (for hierarchical / syntactic decoding)");
 +            }
 +
 +            if (search_algorithm.equals("cky") && include_align_index) {
 +              throw new RuntimeException(
 +                  "include_align_index is currently not supported with cky search");
 +            }
 +
 +          } else if (parameter.equals(normalize_key("reordering-limit"))) {
 +            reordering_limit = Integer.parseInt(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("num-translation-options"))) {
 +            num_translation_options = Integer.parseInt(fds[1]);
 +
 +          } else if (parameter.equals(normalize_key("no-dot-chart"))) {
 +            use_dot_chart = false;
 +
 +          } else if (parameter.equals(normalize_key("moses"))) {
 +            moses = true; // triggers some Moses-specific compatibility options
 +
 +          } else if (parameter.equals(normalize_key("show-weights"))) {
 +            show_weights_and_quit = true;
 +
 +          } else if (parameter.equals(normalize_key("n-best-list"))) {
 +            // for Moses compatibility
 +            String[] tokens = fds[1].split("\\s+");
 +            n_best_file = tokens[0];
 +            if (tokens.length > 1)
 +              topN = Integer.parseInt(tokens[1]);
 +
 +          } else if (parameter.equals(normalize_key("input-file"))) {
 +            // for Moses compatibility
 +            input_file = fds[1];
 +
 +          } else if (parameter.equals(normalize_key("weight-file"))) {
 +            // for Moses, ignore
 +
 +          } else if (parameter.equals(normalize_key("weight-overwrite"))) {
 +            weight_overwrite = fds[1];
 +
 +          } else if (parameter.equals(normalize_key("source-annotations"))) {
 +            // Check source sentence
 +            source_annotations = true;
 +
 +          } else if (parameter.equals(normalize_key("cached-rules-size"))) {
 +            // Check source sentence
 +            cachedRuleSize = Integer.parseInt(fds[1]);
 +          } else if (parameter.equals(normalize_key("lowercase"))) {
 +            lowercase = true;
 +
 +          } else if (parameter.equals(normalize_key("project-case"))) {
 +            project_case = true;
 +
 +          } else {
 +
 +            if (parameter.equals(normalize_key("use-sent-specific-tm"))
 +                || parameter.equals(normalize_key("add-combined-cost"))
 +                || parameter.equals(normalize_key("use-tree-nbest"))
 +                || parameter.equals(normalize_key("use-kenlm"))
 +                || parameter.equals(normalize_key("useCubePrune"))
 +                || parameter.equals(normalize_key("useBeamAndThresholdPrune"))
 +                || parameter.equals(normalize_key("regexp-grammar"))) {
 +              LOG.warn("ignoring deprecated parameter '{}'", fds[0]);
 +
 +            } else {
 +              throw new RuntimeException("FATAL: unknown configuration parameter '" + fds[0] + "'");
 +            }
 +          }
 +
 +          LOG.info("    {} = '{}'", normalize_key(fds[0]), fds[1]);
 +
 +        } else {
 +          /*
 +           * Lines that don't have an equals sign and are not blank lines, empty lines, or comments,
 +           * are feature values, which can be present in this file
 +           */
 +
 +          weights.add(line);
 +        }
 +      }
 +    } finally {
 +      configReader.close();
 +    }
 +  }
 +
 +  /**
 +   * Checks for invalid variable configurations
 +   */
 +  public void sanityCheck() {
 +  }
 +  
 +  /**
 +   * Sets the verbosity level to v (0: OFF; 1: INFO; 2: DEBUG).
 +   * 
 +   * @param v the verbosity level (0, 1, or 2)
 +   */
 +  public void setVerbosity(int v) {
 +    Decoder.VERBOSE = v;
 +    switch (Decoder.VERBOSE) {
 +    case 0:
 +      LogManager.getRootLogger().setLevel(Level.OFF);
 +      break;
 +    case 1:
 +      LogManager.getRootLogger().setLevel(Level.INFO);
 +      break;
 +    case 2:
 +      LogManager.getRootLogger().setLevel(Level.DEBUG);
 +      break;
 +    }
 +  }
 +
 +  /**
 +   * Normalizes parameter names by removing underscores and hyphens and lowercasing. This defines
 +   * equivalence classes on external use of parameter names, permitting arbitrary_under_scores and
 +   * camelCasing in paramter names without forcing the user to memorize them all. Here are some
 +   * examples of equivalent ways to refer to parameter names:
 +   * <pre>
 +   * {pop-limit, poplimit, PopLimit, popLimit, pop_lim_it} {lmfile, lm-file, LM-FILE, lm_file}
 +   * </pre>
 +   * 
 +   * @param text the string to be normalized
 +   * @return normalized key
 +   * 
 +   */
 +  public static String normalize_key(String text) {
 +    return text.replaceAll("[-_]", "").toLowerCase();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
index 9f63cad,0000000..d3913fb
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
@@@ -1,446 -1,0 +1,443 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.Scanner;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.PriorityBlockingQueue;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.joshua.util.Ngram;
 +import org.apache.joshua.util.Regex;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +/**
 + * this class implements: (1) nbest min risk (MBR) reranking using BLEU as a gain funtion.
 + * <p>
 + * This assume that the string is unique in the nbest list In Hiero, due to spurious ambiguity, a
 + * string may correspond to many possible derivations, and ideally the probability of a string
 + * should be the sum of all the derivataions leading to that string. But, in practice, one normally
 + * uses a Viterbi approximation: the probability of a string is its best derivation probability So,
 + * if one want to deal with spurious ambiguity, he/she should do that before calling this class
 + * 
 + * @author Zhifei Li, zhifei.work@gmail.com
 + */
 +public class NbestMinRiskReranker {
 +
 +  private static final Logger LOG = LoggerFactory.getLogger(NbestMinRiskReranker.class);
 +
 +  // TODO: this functionality is not implemented yet; default is to produce 1best without any
 +  // feature scores;
 +  boolean produceRerankedNbest = false;
 +
 +  double scalingFactor = 1.0;
 +
-   static int bleuOrder = 4;
-   static boolean doNgramClip = true;
++  static final int bleuOrder = 4;
++  static final boolean doNgramClip = true;
 +
-   static boolean useGoogleLinearCorpusGain = false;
++  static final boolean useGoogleLinearCorpusGain = false;
 +
-   final PriorityBlockingQueue<RankerResult> resultsQueue =
-       new PriorityBlockingQueue<RankerResult>();
++  final PriorityBlockingQueue<RankerResult> resultsQueue = new PriorityBlockingQueue<>();
 +
 +  public NbestMinRiskReranker(boolean produceRerankedNbest, double scalingFactor) {
 +    this.produceRerankedNbest = produceRerankedNbest;
 +    this.scalingFactor = scalingFactor;
 +  }
 +
 +
 +  public String processOneSent(List<String> nbest, int sentID) {
 +    LOG.info("Now process sentence {}", sentID);
 +
 +    // step-0: preprocess
 +    // assumption: each hyp has a formate:
 +    // "sent_id ||| hyp_itself ||| feature scores ||| linear-combination-of-feature-scores(this should be logP)"
 +
 +    /* Quit if you find an empty hypothesis. */
 +    if (nbest.size() == 1) {
 +      String[] fields = Regex.threeBarsWithSpace.split(nbest.get(0));
 +      if (fields[1].equals("") || Regex.spaces.matches(fields[1])) {
 +        LOG.warn("-> sentence is empty");
 +        return "";
 +      }
 +    } 
 +
-     List<String> hypsItself = new ArrayList<String>();
++    List<String> hypsItself = new ArrayList<>();
 +    // ArrayList<String> l_feat_scores = new ArrayList<String>();
-     List<Double> baselineScores = new ArrayList<Double>(); // linear combination of all baseline
++    List<Double> baselineScores = new ArrayList<>(); // linear combination of all baseline
 +                                                           // features
-     List<HashMap<String, Integer>> ngramTbls = new ArrayList<HashMap<String, Integer>>();
-     List<Integer> sentLens = new ArrayList<Integer>();
++    List<HashMap<String, Integer>> ngramTbls = new ArrayList<>();
++    List<Integer> sentLens = new ArrayList<>();
 +
 +    for (String hyp : nbest) {
 +      String[] fds = Regex.threeBarsWithSpace.split(hyp);
 +      int tSentID = Integer.parseInt(fds[0]);
 +      if (sentID != tSentID) {
 +        throw new RuntimeException("sentence_id does not match");
 +      }
 +      String hypothesis = (fds.length >= 4) ? fds[1] : "";
 +      hypsItself.add(hypothesis);
 +
 +      String[] words = Regex.spaces.split(hypothesis);
 +      sentLens.add(words.length);
 +
-       HashMap<String, Integer> ngramTbl = new HashMap<String, Integer>();
++      HashMap<String, Integer> ngramTbl = new HashMap<>();
 +      Ngram.getNgrams(ngramTbl, 1, bleuOrder, words);
 +      ngramTbls.add(ngramTbl);
 +
 +      // l_feat_scores.add(fds[2]);
 +
 +      // The value of finalIndex is expected to be 3,
 +      // unless the hyp_itself is empty,
 +      // in which case finalIndex will be 2.
 +      int finalIndex = fds.length - 1;
 +      baselineScores.add(Double.parseDouble(fds[finalIndex]));
 +
 +    }
 +
 +    // step-1: get normalized distribution
 +
 +    /**
 +     * value in baselineScores will be changed to normalized probability
 +     * */
 +    computeNormalizedProbs(baselineScores, scalingFactor);
 +
-     List<Double> normalizedProbs = baselineScores;
- 
 +    // === required by google linear corpus gain
 +    HashMap<String, Double> posteriorCountsTbl = null;
 +    if (useGoogleLinearCorpusGain) {
-       posteriorCountsTbl = new HashMap<String, Double>();
-       getGooglePosteriorCounts(ngramTbls, normalizedProbs, posteriorCountsTbl);
++      posteriorCountsTbl = new HashMap<>();
++      getGooglePosteriorCounts(ngramTbls, baselineScores, posteriorCountsTbl);
 +    }
 +
 +
 +    // step-2: rerank the nbest
 +    /**
 +     * TODO: zhifei: now the re-ranking takes O(n^2) where n is the size of the nbest. But, we can
 +     * significantly speed up this (leadding to O(n)) by first estimating a model on nbest, and then
 +     * rerank the nbest using the estimated model.
 +     * */
 +    double bestGain = -1000000000;// set as worst gain
 +    String bestHyp = null;
-     List<Double> gains = new ArrayList<Double>();
++    List<Double> gains = new ArrayList<>();
 +    for (int i = 0; i < hypsItself.size(); i++) {
 +      String curHyp = hypsItself.get(i);
 +      int curHypLen = sentLens.get(i);
 +      HashMap<String, Integer> curHypNgramTbl = ngramTbls.get(i);
 +      // double cur_gain = computeGain(cur_hyp, l_hyp_itself, l_normalized_probs);
 +      double curGain = 0;
 +      if (useGoogleLinearCorpusGain) {
 +        curGain = computeExpectedLinearCorpusGain(curHypLen, curHypNgramTbl, posteriorCountsTbl);
 +      } else {
 +        curGain =
-             computeExpectedGain(curHypLen, curHypNgramTbl, ngramTbls, sentLens, normalizedProbs);
++            computeExpectedGain(curHypLen, curHypNgramTbl, ngramTbls, sentLens, baselineScores);
 +      }
 +
 +      gains.add(curGain);
 +      if (i == 0 || curGain > bestGain) { // maximize
 +        bestGain = curGain;
 +        bestHyp = curHyp;
 +      }
 +    }
 +
 +    // step-3: output the 1best or nbest
 +    if (this.produceRerankedNbest) {
 +      // TOTO: sort the list and write the reranked nbest; Use Collections.sort(List list,
 +      // Comparator c)
 +    } else {
 +      /*
 +       * this.out.write(best_hyp); this.out.write("\n"); out.flush();
 +       */
 +    }
 +
 +    LOG.info("best gain: {}", bestGain);
 +    if (null == bestHyp) {
 +      throw new RuntimeException("mbr reranked one best is null, must be wrong");
 +    }
 +    return bestHyp;
 +  }
 +
 +
 +  /**
 +   * based on a list of log-probabilities in nbestLogProbs, obtain a normalized distribution, and
 +   * put the normalized probability (real value in [0,1]) into nbestLogProbs
 +   * 
 +   * @param nbestLogProbs a {@link java.util.List} of {@link java.lang.Double} representing nbestLogProbs
 +   * @param scalingFactor double value representing scaling factor
 +   */
 +  // get a normalized distributeion and put it back to nbestLogProbs
 +  static public void computeNormalizedProbs(List<Double> nbestLogProbs, double scalingFactor) {
 +
 +    // === get noralization constant, remember features, remember the combined linear score
 +    double normalizationConstant = Double.NEGATIVE_INFINITY;// log-semiring
 +
 +    for (double logp : nbestLogProbs) {
 +      normalizationConstant = addInLogSemiring(normalizationConstant, logp * scalingFactor, 0);
 +    }
 +    // System.out.println("normalization_constant (logP) is " + normalization_constant);
 +
 +    // === get normalized prob for each hyp
 +    double tSum = 0;
 +    for (int i = 0; i < nbestLogProbs.size(); i++) {
 +
 +      double normalizedProb =
 +          Math.exp(nbestLogProbs.get(i) * scalingFactor - normalizationConstant);
 +      tSum += normalizedProb;
 +      nbestLogProbs.set(i, normalizedProb);
 +
 +      if (Double.isNaN(normalizedProb)) {
 +        throw new RuntimeException("prob is NaN, must be wrong\nnbest_logps.get(i): "
 +            + nbestLogProbs.get(i) + "; scaling_factor: " + scalingFactor
 +            + "; normalization_constant:" + normalizationConstant);
 +      }
 +      // logger.info("probability: " + normalized_prob);
 +    }
 +
 +    // sanity check
 +    if (Math.abs(tSum - 1.0) > 1e-4) {
 +      throw new RuntimeException("probabilities not sum to one, must be wrong");
 +    }
 +
 +  }
 +
 +
 +  // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
 +  // curHyp: e
 +  // trueHyp: e'
 +  public double computeExpectedGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
 +      List<HashMap<String, Integer>> ngramTbls, List<Integer> sentLens, List<Double> nbestProbs) {
 +
 +    // ### get noralization constant, remember features, remember the combined linear score
 +    double gain = 0;
 +
 +    for (int i = 0; i < nbestProbs.size(); i++) {
 +      HashMap<String, Integer> trueHypNgramTbl = ngramTbls.get(i);
 +      double trueProb = nbestProbs.get(i);
 +      int trueLen = sentLens.get(i);
 +      gain +=
 +          trueProb
 +              * BLEU.computeSentenceBleu(trueLen, trueHypNgramTbl, curHypLen, curHypNgramTbl,
 +                  doNgramClip, bleuOrder);
 +    }
 +    // System.out.println("Gain is " + gain);
 +    return gain;
 +  }
 +
 +  // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
 +  // curHyp: e
 +  // trueHyp: e'
 +  static public double computeExpectedGain(String curHyp, List<String> nbestHyps,
 +      List<Double> nbestProbs) {
 +    // ### get noralization constant, remember features, remember the combined linear score
 +    double gain = 0;
 +
 +    for (int i = 0; i < nbestHyps.size(); i++) {
 +      String trueHyp = nbestHyps.get(i);
 +      double trueProb = nbestProbs.get(i);
 +      gain += trueProb * BLEU.computeSentenceBleu(trueHyp, curHyp, doNgramClip, bleuOrder);
 +    }
 +    // System.out.println("Gain is " + gain);
 +    return gain;
 +  }
 +
 +  void getGooglePosteriorCounts(List<HashMap<String, Integer>> ngramTbls,
 +      List<Double> normalizedProbs, HashMap<String, Double> posteriorCountsTbl) {
 +    // TODO
 +  }
 +
 +  double computeExpectedLinearCorpusGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
 +      HashMap<String, Double> posteriorCountsTbl) {
 +    // TODO
 +    double[] thetas = {-1, 1, 1, 1, 1};
 +
 +    double res = 0;
 +    res += thetas[0] * curHypLen;
 +    for (Entry<String, Integer> entry : curHypNgramTbl.entrySet()) {
 +      String key = entry.getKey();
 +      String[] tem = Regex.spaces.split(key);
 +
 +      double post_prob = posteriorCountsTbl.get(key);
 +      res += entry.getValue() * post_prob * thetas[tem.length];
 +    }
 +    return res;
 +  }
 +
 +  // OR: return Math.log(Math.exp(x) + Math.exp(y));
 +  static private double addInLogSemiring(double x, double y, int addMode) {// prevent over-flow
 +    if (addMode == 0) { // sum
 +      if (x == Double.NEGATIVE_INFINITY) {// if y is also n-infinity, then return n-infinity
 +        return y;
 +      }
 +      if (y == Double.NEGATIVE_INFINITY) {
 +        return x;
 +      }
 +
 +      if (y <= x) {
 +        return x + Math.log(1 + Math.exp(y - x));
 +      } else {
 +        return y + Math.log(1 + Math.exp(x - y));
 +      }
 +    } else if (addMode == 1) { // viter-min
 +      return (x <= y) ? x : y;
 +    } else if (addMode == 2) { // viter-max
 +      return (x >= y) ? x : y;
 +    } else {
 +      throw new RuntimeException("invalid add mode");
 +    }
 +  }
 +
 +
 +
 +  public static void main(String[] args) throws IOException {
 +
 +    // If you don't know what to use for scaling factor, try using 1
 +
 +    if (args.length < 2) {
 +      String msg = "usage: java NbestMinRiskReranker <produce_reranked_nbest> <scaling_factor> "
 +          + "[numThreads]";
 +      System.err.println(msg);
 +      LOG.error(msg);
 +      return;
 +    }
 +    long startTime = System.currentTimeMillis();
 +    boolean produceRerankedNbest = Boolean.valueOf(args[0].trim());
 +    double scalingFactor = Double.parseDouble(args[1].trim());
 +    int numThreads = (args.length > 2) ? Integer.parseInt(args[2].trim()) : 1;
 +
 +
 +    NbestMinRiskReranker mbrReranker =
 +        new NbestMinRiskReranker(produceRerankedNbest, scalingFactor);
 +
 +    LOG.info("Running mbr reranking");
 +
 +    int oldSentID = -1;
-     List<String> nbest = new ArrayList<String>();
++    List<String> nbest = new ArrayList<>();
 +
 +    Scanner scanner = new Scanner(System.in, "UTF-8");
 +
 +    if (numThreads == 1) {
 +
 +      while (scanner.hasNextLine()) {
 +        String line = scanner.nextLine();
 +        String[] fds = Regex.threeBarsWithSpace.split(line);
 +        int newSentID = Integer.parseInt(fds[0]);
 +        if (oldSentID != -1 && oldSentID != newSentID) {
 +          if (nbest.size() > 0) {
 +            String best_hyp = mbrReranker.processOneSent(nbest, oldSentID);// nbest: list of unique
 +                                                                           // strings
 +            System.out.println(best_hyp);
 +          } else {
 +            System.out.println();
 +          }
 +          nbest.clear();
 +        }
 +        oldSentID = newSentID;
 +        if (!fds[1].matches("^\\s*$")) nbest.add(line);
 +      }
 +
 +      // last nbest
 +      if (oldSentID >= 0) {
 +        String bestHyp = mbrReranker.processOneSent(nbest, oldSentID);
 +        System.out.println(bestHyp);
 +        nbest.clear();
 +      }
 +
 +    } else {
 +
 +      ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
 +
 +      while (scanner.hasNextLine()) {
 +        String line = scanner.nextLine();
 +        String[] fds = Regex.threeBarsWithSpace.split(line);
 +        int newSentID = Integer.parseInt(fds[0]);
 +        if (oldSentID != -1 && oldSentID != newSentID) {
 +
 +          threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
 +
 +          nbest.clear();
 +        }
 +        oldSentID = newSentID;
 +        nbest.add(line);
 +      }
 +
 +      // last nbest
 +      threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
 +      nbest.clear();
 +
 +      threadPool.shutdown();
 +
 +      try {
 +        threadPool.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
 +
 +        while (!mbrReranker.resultsQueue.isEmpty()) {
 +          RankerResult result = mbrReranker.resultsQueue.remove();
 +          String best_hyp = result.toString();
 +          System.out.println(best_hyp);
 +        }
 +      } catch (InterruptedException e) {
 +        LOG.error(e.getMessage(), e);
 +      }
 +    }
 +    
 +    scanner.close();
 +
 +    LOG.info("Total running time (seconds) is {} ",
 +        (System.currentTimeMillis() - startTime) / 1000.0);
 +  }
 +
 +  private class RankerTask implements Runnable {
 +
 +    final List<String> nbest;
 +    final int sentID;
 +
 +    RankerTask(final List<String> nbest, final int sentID) {
-       this.nbest = new ArrayList<String>(nbest);
++      this.nbest = new ArrayList<>(nbest);
 +      this.sentID = sentID;
 +    }
 +
 +    public void run() {
 +      String result = processOneSent(nbest, sentID);
 +      resultsQueue.add(new RankerResult(result, sentID));
 +    }
 +
 +  }
 +
 +  private static class RankerResult implements Comparable<RankerResult> {
 +    final String result;
 +    final Integer sentenceNumber;
 +
 +    RankerResult(String result, int sentenceNumber) {
 +      this.result = result;
 +      this.sentenceNumber = sentenceNumber;
 +    }
 +
 +    public int compareTo(RankerResult o) {
 +      return sentenceNumber.compareTo(o.sentenceNumber);
 +    }
 +
 +    public String toString() {
 +      return result;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
index 0e167d1,0000000..9f32d31
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
@@@ -1,154 -1,0 +1,154 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import java.util.List;
 +import java.util.Map;
 +
++import org.apache.joshua.decoder.io.DeNormalize;
 +import org.apache.joshua.decoder.segment_file.Sentence;
 +import org.apache.joshua.decoder.segment_file.Token;
 +import org.apache.joshua.util.FormatUtils;
 +
 +/**
 + * A StructuredTranslation instance provides a more structured access to
 + * translation results than the string-based Translation class.
 + * This is useful if the decoder is encapsulated in a larger project, instead
 + * of simply writing to a file or stdout.
 + * StructuredTranslation encodes all relevant information about a derivation,
 + * namely output string, tokens, score, features, and word alignment.
 + * 
 + * @author fhieber
 + */
 +public class StructuredTranslation {
 +  
 +  private final Sentence sourceSentence;
 +  private final String translationString;
 +  private final List<String> translationTokens;
 +  private final float translationScore;
 +  private final List<List<Integer>> translationWordAlignments;
 +  private final Map<String,Float> translationFeatures;
 +  private final float extractionTime;
 +  
 +  public StructuredTranslation(
 +      final Sentence sourceSentence,
 +      final String translationString,
 +      final List<String> translationTokens,
 +      final float translationScore,
 +      final List<List<Integer>> translationWordAlignments,
 +      final Map<String,Float> translationFeatures,
 +      final float extractionTime) {
 +    this.sourceSentence = sourceSentence;
 +    this.translationString = translationString;
 +    this.translationTokens = translationTokens;
 +    this.translationScore = translationScore;
 +    this.translationWordAlignments = translationWordAlignments;
 +    this.translationFeatures = translationFeatures;
 +    this.extractionTime = extractionTime;
 +  }
 +  
 +  public Sentence getSourceSentence() {
 +    return sourceSentence;
 +  }
 +
 +  public int getSentenceId() {
 +    return sourceSentence.id();
 +  }
 +
 +  /**
 +   * Produces the raw translation hypothesis (still tokenized).
 +   * 
 +   * @return the raw translation hypothesis
 +   */
 +  public String getTranslationString() {
 +    return translationString;
 +  }
 +  
 +  /**
 +   * Returns the output string formatted according to JoshuaConfiguration.output_format.
 +   * 
 +   * @return the formatted string
 +   */
 +  public String getFormattedTranslationString() {
-     throw new RuntimeException("Not yet implemented");
++    return DeNormalize.processSingleLine(maybeProjectCase(getTranslationString()));
 +  }
 +
 +  public List<String> getTranslationTokens() {
 +    return translationTokens;
 +  }
 +
 +  public float getTranslationScore() {
 +    return translationScore;
 +  }
 +
 +  /**
 +   * Returns a list of target to source alignments.
 +   * 
 +   * @return a list of target to source alignments
 +   */
 +  public List<List<Integer>> getTranslationWordAlignments() {
 +    return translationWordAlignments;
 +  }
 +  
 +  public Map<String,Float> getTranslationFeatures() {
 +    return translationFeatures;
 +  }
 +  
 +  /**
 +   * Time taken to build output information from the hypergraph.
 +   * @return the time taken to build output information from the hypergraph
 +   */
 +  public Float getExtractionTime() {
 +    return extractionTime;
 +  }
 +  
 +  /**
 +   * If requested, projects source-side lettercase to target, and appends the alignment from
 +   * to the source-side sentence in ||s.
 +   * 
-    * @param hypothesis todo
-    * @param state todo
-    * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
++   * @param hypothesis the string hypothesis
++   * @return source-side lettercase to target, and appends the alignment from to the source-side sentence
 +   */
 +  private String maybeProjectCase(String hypothesis) {
 +    String output = hypothesis;
 +
 +    JoshuaConfiguration config = sourceSentence.config;
 +    if (config.project_case) {
 +      String[] tokens = hypothesis.split("\\s+");
 +      List<List<Integer>> points = getTranslationWordAlignments();
 +      for (int i = 0; i < points.size(); i++) {
 +        List<Integer> target = points.get(i);
 +        for (int source: target) {
 +          Token token = sourceSentence.getTokens().get(source + 1); // skip <s>
 +          String annotation = "";
 +          if (token != null && token.getAnnotation("lettercase") != null)
 +            annotation = token.getAnnotation("lettercase");
 +          if (source != 0 && annotation.equals("upper"))
 +            tokens[i] = FormatUtils.capitalize(tokens[i]);
 +          else if (annotation.equals("all-upper"))
 +            tokens[i] = tokens[i].toUpperCase();
 +        }
 +      }
 +
 +      output = String.join(" ",  tokens);
 +    }
 +
 +    return output;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
index ce4f32c,0000000..6453bd1
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
@@@ -1,116 -1,0 +1,116 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import static java.util.Arrays.asList;
 +import static java.util.Collections.emptyList;
 +import static java.util.Collections.emptyMap;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
 +import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
 +
 +import java.util.List;
 +
 +import org.apache.joshua.decoder.ff.FeatureFunction;
 +import org.apache.joshua.decoder.hypergraph.HyperGraph;
 +import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
 +import org.apache.joshua.decoder.segment_file.Sentence;
++import org.apache.joshua.decoder.segment_file.Token;
++import org.apache.joshua.util.FormatUtils;
 +
 +/**
 + * This factory provides methods to create StructuredTranslation objects
 + * from either Viterbi derivations or KBest derivations.
 + * 
 + * @author fhieber
 + */
 +public class StructuredTranslationFactory {
 +  
 +  /**
 +   * Returns a StructuredTranslation instance from the Viterbi derivation.
 +   * 
 +   * @param sourceSentence the source sentence
 +   * @param hypergraph the hypergraph object
 +   * @param featureFunctions the list of active feature functions
 +   * @return A StructuredTranslation object representing the Viterbi derivation.
 +   */
 +  public static StructuredTranslation fromViterbiDerivation(
 +      final Sentence sourceSentence,
 +      final HyperGraph hypergraph,
 +      final List<FeatureFunction> featureFunctions) {
 +    final long startTime = System.currentTimeMillis();
 +    final String translationString = removeSentenceMarkers(getViterbiString(hypergraph));
 +    return new StructuredTranslation(
 +        sourceSentence,
 +        translationString,
 +        extractTranslationTokens(translationString),
 +        extractTranslationScore(hypergraph),
 +        getViterbiWordAlignmentList(hypergraph),
 +        getViterbiFeatures(hypergraph, featureFunctions, sourceSentence).toStringMap(),
 +        (System.currentTimeMillis() - startTime) / 1000.0f);
 +  }
 +  
 +  /**
 +   * Returns a StructuredTranslation from an empty decoder output
 +   * @param sourceSentence the source sentence
 +   * @return a StructuredTranslation object
 +   */
 +  public static StructuredTranslation fromEmptyOutput(final Sentence sourceSentence) {
 +        return new StructuredTranslation(
 +                sourceSentence, "", emptyList(), 0, emptyList(), emptyMap(), 0f);
 +      }
 +  
 +  /**
 +   * Returns a StructuredTranslation instance from a KBest DerivationState. 
 +   * @param sourceSentence Sentence object representing the source.
 +   * @param derivationState the KBest DerivationState.
 +   * @return A StructuredTranslation object representing the derivation encoded by derivationState.
 +   */
 +  public static StructuredTranslation fromKBestDerivation(
 +      final Sentence sourceSentence,
 +      final DerivationState derivationState) {
 +    final long startTime = System.currentTimeMillis();
 +    final String translationString = removeSentenceMarkers(derivationState.getHypothesis());
 +    return new StructuredTranslation(
 +        sourceSentence,
 +        translationString,
 +        extractTranslationTokens(translationString),
 +        derivationState.getModelCost(),
 +        derivationState.getWordAlignmentList(),
 +        derivationState.getFeatures().toStringMap(),
 +        (System.currentTimeMillis() - startTime) / 1000.0f);
 +  }
 +  
 +  private static float extractTranslationScore(final HyperGraph hypergraph) {
 +    if (hypergraph == null) {
 +      return 0;
 +    } else {
 +      return hypergraph.goalNode.getScore();
 +    }
 +  }
 +  
 +  private static List<String> extractTranslationTokens(final String translationString) {
 +    if (translationString.isEmpty()) {
 +      return emptyList();
 +    } else {
 +      return asList(translationString.split("\\s+"));
 +    }
 +  }
-   
- 
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
index cdda63e,0000000..260b0ac
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
@@@ -1,237 -1,0 +1,239 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import static java.util.Arrays.asList;
 +import static org.apache.joshua.decoder.StructuredTranslationFactory.fromViterbiDerivation;
 +import static org.apache.joshua.decoder.ff.FeatureMap.hashFeature;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
 +import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignments;
 +import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
 +
 +import java.io.BufferedWriter;
 +import java.io.IOException;
 +import java.io.StringWriter;
++import java.util.Collections;
 +import java.util.List;
 +
 +import org.apache.joshua.decoder.ff.FeatureFunction;
 +import org.apache.joshua.decoder.ff.FeatureVector;
 +import org.apache.joshua.decoder.ff.lm.StateMinimizingLanguageModel;
 +import org.apache.joshua.decoder.hypergraph.HyperGraph;
 +import org.apache.joshua.decoder.hypergraph.KBestExtractor;
 +import org.apache.joshua.decoder.io.DeNormalize;
 +import org.apache.joshua.decoder.segment_file.Sentence;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +/**
 + * This class represents translated input objects (sentences or lattices). It is aware of the source
 + * sentence and id and contains the decoded hypergraph. Translation objects are returned by
 + * DecoderThread instances to the InputHandler, where they are assembled in order for output.
 + * 
 + * @author Matt Post post@cs.jhu.edu
 + * @author Felix Hieber fhieber@amazon.com
 + */
 +
 +public class Translation {
 +  private static final Logger LOG = LoggerFactory.getLogger(Translation.class);
-   private Sentence source;
++  private final Sentence source;
 +
 +  /**
 +   * This stores the output of the translation so we don't have to hold onto the hypergraph while we
 +   * wait for the outputs to be assembled.
 +   */
 +  private String output = null;
 +
 +  /**
 +   * Stores the list of StructuredTranslations.
 +   * If joshuaConfig.topN == 0, will only contain the Viterbi translation.
 +   * Else it will use KBestExtractor to populate this list.
 +   */
 +  private List<StructuredTranslation> structuredTranslations = null;
 +  
 +  public Translation(Sentence source, HyperGraph hypergraph, 
 +      List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) {
 +    this.source = source;
 +    
 +    /**
 +     * Structured output from Joshua provides a way to programmatically access translation results
 +     * from downstream applications, instead of writing results as strings to an output buffer.
 +     */
 +    if (joshuaConfiguration.use_structured_output) {
 +      
 +      if (joshuaConfiguration.topN == 0) {
 +        /*
 +         * Obtain Viterbi StructuredTranslation
 +         */
 +        StructuredTranslation translation = fromViterbiDerivation(source, hypergraph, featureFunctions);
 +        this.output = translation.getTranslationString();
-         structuredTranslations = asList(translation);
++        structuredTranslations = Collections.singletonList(translation);
 +        
 +      } else {
 +        /*
 +         * Get K-Best list of StructuredTranslations
 +         */
 +        final KBestExtractor kBestExtractor = new KBestExtractor(source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
 +        structuredTranslations = kBestExtractor.KbestExtractOnHG(hypergraph, joshuaConfiguration.topN);
 +        if (structuredTranslations.isEmpty()) {
-             structuredTranslations = asList(StructuredTranslationFactory.fromEmptyOutput(source));
++            structuredTranslations = Collections
++                .singletonList(StructuredTranslationFactory.fromEmptyOutput(source));
 +            this.output = "";
 +        } else {
 +            this.output = structuredTranslations.get(0).getTranslationString();
 +        }
 +        // TODO: We omit the BLEU rescoring for now since it is not clear whether it works at all and what the desired output is below.
 +      }
 +
 +    } else {
 +
 +      StringWriter sw = new StringWriter();
 +      BufferedWriter out = new BufferedWriter(sw);
 +
 +      try {
 +        
 +        if (hypergraph != null) {
 +          
 +          long startTime = System.currentTimeMillis();
 +
 +          if (joshuaConfiguration.topN == 0) {
 +
 +            /* construct Viterbi output */
 +            final String best = getViterbiString(hypergraph);
 +
 +            LOG.info("Translation {}: {} {}", source.id(), hypergraph.goalNode.getScore(), best);
 +
 +            /*
 +             * Setting topN to 0 turns off k-best extraction, in which case we need to parse through
 +             * the output-string, with the understanding that we can only substitute variables for the
 +             * output string, sentence number, and model score.
 +             */
 +            String translation = joshuaConfiguration.outputFormat
 +                .replace("%s", removeSentenceMarkers(best))
 +                .replace("%S", DeNormalize.processSingleLine(best))
 +                .replace("%c", String.format("%.3f", hypergraph.goalNode.getScore()))
 +                .replace("%i", String.format("%d", source.id()));
 +
 +            if (joshuaConfiguration.outputFormat.contains("%a")) {
 +              translation = translation.replace("%a", getViterbiWordAlignments(hypergraph));
 +            }
 +
 +            if (joshuaConfiguration.outputFormat.contains("%f")) {
 +              final FeatureVector features = getViterbiFeatures(hypergraph, featureFunctions, source);
 +              translation = translation.replace("%f", features.textFormat());
 +            }
 +
 +            out.write(translation);
 +            out.newLine();
 +
 +          } else {
 +
 +            final KBestExtractor kBestExtractor = new KBestExtractor(
 +                source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
 +            kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
 +
 +            if (joshuaConfiguration.rescoreForest) {
 +              final int bleuFeatureHash = hashFeature("BLEU");
 +              Decoder.weights.add(bleuFeatureHash, joshuaConfiguration.rescoreForestWeight);
 +              kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
 +
 +              Decoder.weights.add(bleuFeatureHash, -joshuaConfiguration.rescoreForestWeight);
 +              kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
 +            }
 +          }
 +
 +          float seconds = (float) (System.currentTimeMillis() - startTime) / 1000.0f;
 +          LOG.info("Input {}: {}-best extraction took {} seconds", id(),
 +              joshuaConfiguration.topN, seconds);
 +
 +        } else {
 +          
 +          // Failed translations and blank lines get empty formatted outputs
 +          out.write(getFailedTranslationOutput(source, joshuaConfiguration));
 +          out.newLine();
 +          
 +        }
 +
 +        out.flush();
 +        
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      this.output = sw.toString();
 +
 +    }
 +    
 +    // remove state from StateMinimizingLanguageModel instances in features.
 +    destroyKenLMStates(featureFunctions);
 +
 +  }
 +
 +  public Sentence getSourceSentence() {
 +    return this.source;
 +  }
 +
 +  public int id() {
 +    return source.id();
 +  }
 +
 +  @Override
 +  public String toString() {
 +    return output;
 +  }
 +  
 +  private String getFailedTranslationOutput(final Sentence source, final JoshuaConfiguration joshuaConfiguration) {
 +    return joshuaConfiguration.outputFormat
 +        .replace("%s", source.source())
 +        .replace("%e", "")
 +        .replace("%S", "")
 +        .replace("%t", "()")
 +        .replace("%i", Integer.toString(source.id()))
 +        .replace("%f", "")
 +        .replace("%c", "0.000");
 +  }
 +  
 +  /**
 +   * Returns the StructuredTranslations
 +   * if JoshuaConfiguration.use_structured_output == True.
 +   * @throws RuntimeException if JoshuaConfiguration.use_structured_output == False.
 +   * @return List of StructuredTranslations.
 +   */
 +  public List<StructuredTranslation> getStructuredTranslations() {
 +    if (structuredTranslations == null) {
 +      throw new RuntimeException(
 +          "No StructuredTranslation objects created. You should set JoshuaConfigration.use_structured_output = true");
 +    }
 +    return structuredTranslations;
 +  }
 +  
 +  /**
 +   * KenLM hack. If using KenLMFF, we need to tell KenLM to delete the pool used to create chart
 +   * objects for this sentence.
 +   */
 +  private void destroyKenLMStates(final List<FeatureFunction> featureFunctions) {
 +    for (FeatureFunction feature : featureFunctions) {
 +      if (feature instanceof StateMinimizingLanguageModel) {
 +        ((StateMinimizingLanguageModel) feature).destroyPool(getSourceSentence().id());
 +        break;
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
index e607225,0000000..1eb859a
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
@@@ -1,158 -1,0 +1,158 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder;
 +
 +import java.util.Iterator;
 +import java.util.LinkedList;
 +import org.apache.joshua.decoder.io.TranslationRequestStream;
 +
 +/**
 + * This class represents a streaming sequence of translations. It is returned by the main entry
 + * point to the Decoder object, the call to decodeAll. The translations here are parallel to the
 + * input sentences in the corresponding TranslationRequest object. Because of parallelization, the
 + * translated sentences might be computed out of order. Each Translation is sent to this
 + * Translations object by a DecoderThreadRunner via the record() function, which places the
 + * Translation in the right place. When the next translation in a sequence is available, next() is
 + * notified.
 + * 
 + * @author Matt Post post@cs.jhu.edu
 + */
 +public class Translations implements Iterator<Translation>, Iterable<Translation> {
 +
 +  /* The source sentences to be translated. */
 +  private TranslationRequestStream request = null;
 +
 +  /*
 +   * This records the index of the sentence at the head of the underlying list. The iterator's
 +   * next() blocks when the value at this position in the translations LinkedList is null.
 +   */
 +  private int currentID = 0;
 +
 +  /* The set of translated sentences. */
 +  private LinkedList<Translation> translations = null;
 +
 +  private boolean spent = false;
 +
 +  private Translation nextTranslation;
 +
 +  public Translations(TranslationRequestStream request) {
 +    this.request = request;
-     this.translations = new LinkedList<Translation>();
++    this.translations = new LinkedList<>();
 +  }
 +
 +  /**
 +   * This is called when null is received from the TranslationRequest, indicating that there are no
 +   * more input sentences to translated. That in turn means that the request size will no longer
 +   * grow. We then notify any waiting thread if the last ID we've processed is the last one, period.
 +   */
 +  public void finish() {
 +    synchronized (this) {
 +      spent = true;
 +      if (currentID == request.size()) {
 +        this.notifyAll();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * This is called whenever a translation is completed by one of the decoder threads. There may be
 +   * a current output thread waiting for the current translation, which is determined by checking if
 +   * the ID of the translation is the same as the one being waited for (currentID). If so, the
 +   * thread waiting for it is notified.
 +   * 
 +   * @param translation a translated input object
 +   */
 +  public void record(Translation translation) {
 +    synchronized (this) {
 +
 +      /* Pad the set of translations with nulls to accommodate the new translation. */
 +      int offset = translation.id() - currentID;
 +      while (offset >= translations.size())
 +        translations.add(null);
 +      translations.set(offset, translation);
 +
 +      /*
 +       * If the id of the current translation is at the head of the list (first element), then we
 +       * have the next Translation to be return, and we should notify anyone waiting on next(),
 +       * which will then remove the item and increment the currentID.
 +       */
 +      if (translation.id() == currentID) {
 +        this.notify();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Returns the next Translation, blocking if necessary until it's available, since the next
 +   * Translation might not have been produced yet.
 +   * 
 +   * @return first element from the list of {@link org.apache.joshua.decoder.Translation}'s
 +   */
 +  @Override
 +  public Translation next() {
 +    synchronized(this) {
 +      if (this.hasNext()) {
 +        Translation t = this.nextTranslation;
 +        this.nextTranslation = null;
 +        return t;
 +      }
 +      
 +      return null;
 +    }
 +  }
 +   
 +  @Override
 +  public boolean hasNext() {
 +    synchronized (this) {
 +
 +      if (nextTranslation != null)
 +        return true;
 +      
 +      /*
 +       * If there are no more input sentences, and we've already distributed what we then know is
 +       * the last one, we're done.
 +       */
 +      if (spent && currentID == request.size())
 +        return false;
 +
 +      /*
 +       * Otherwise, there is another sentence. If it's not available already, we need to wait for
 +       * it.
 +       */
 +      if (translations.size() == 0 || translations.peek() == null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {
 +          // TODO Auto-generated catch block
 +          e.printStackTrace();
 +        }
 +      }
 +
 +      /* We now have the sentence and can return it. */
 +      currentID++;
 +      this.nextTranslation = translations.poll();
 +      return this.nextTranslation != null;
 +    }
 +  }
 +
 +  @Override
 +  public Iterator<Translation> iterator() {
 +    return this;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
index 10b9200,0000000..cfcd06b
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
@@@ -1,294 -1,0 +1,294 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *  http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.joshua.decoder.chart_parser;
 +
 +import static com.google.common.base.Preconditions.checkNotNull;
 +
 +import java.util.ArrayList;
 +import java.util.HashMap;
 +import java.util.Iterator;
 +import java.util.LinkedHashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.Map.Entry;
 +
 +import org.apache.joshua.decoder.ff.FeatureFunction;
 +import org.apache.joshua.decoder.ff.state_maintenance.DPState;
 +import org.apache.joshua.decoder.ff.tm.Rule;
 +import org.apache.joshua.decoder.hypergraph.HGNode;
 +import org.apache.joshua.decoder.hypergraph.HyperEdge;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +/**
 + * this class implement functions: (1) combine small itesm into larger ones using rules, and create
 + * items and hyper-edges to construct a hyper-graph, (2) evaluate model score for items, (3)
 + * cube-pruning Note: Bin creates Items, but not all Items will be used in the hyper-graph
 + * 
 + * @author Matt Post <po...@cs.jhu.edu>
 + * @author Zhifei Li, <zh...@gmail.com>
 + */
 +class Cell {
 +
 +  // ===============================================================
 +  // Static fields
 +  // ===============================================================
 +  private static final Logger LOG = LoggerFactory.getLogger(Cell.class);
 +
 +
 +  // The chart this cell belongs to
 +  private Chart chart = null;
 +
 +  // The top-level (goal) symbol
-   private int goalSymbol;
++  private final int goalSymbol;
 +
 +  // to maintain uniqueness of nodes
-   private HashMap<HGNode.Signature, HGNode> nodesSigTbl = new LinkedHashMap<HGNode.Signature, HGNode>();
++  private final HashMap<HGNode.Signature, HGNode> nodesSigTbl = new LinkedHashMap<>();
 +
 +  // signature by lhs
-   private Map<Integer, SuperNode> superNodesTbl = new HashMap<Integer, SuperNode>();
++  private final Map<Integer, SuperNode> superNodesTbl = new HashMap<>();
 +
 +  /**
 +   * sort values in nodesSigTbl, we need this list when necessary
 +   */
 +  private List<HGNode> sortedNodes = null;
 +
 +
 +  // ===============================================================
 +  // Constructor
 +  // ===============================================================
 +
 +  public Cell(Chart chart, int goalSymID) {
 +    this.chart = chart;
 +    this.goalSymbol = goalSymID;
 +  }
 +
 +  public Cell(Chart chart, int goal_sym_id, int constraint_symbol_id) {
 +    this(chart, goal_sym_id);
 +  }
 +
 +  // ===============================================================
 +  // Package-protected methods
 +  // ===============================================================
 +  
 +  public Set<Integer> getKeySet() {
 +    return superNodesTbl.keySet();
 +  }
 +  
 +  public SuperNode getSuperNode(int lhs) {
 +    return superNodesTbl.get(lhs);
 +  }
 +
 +  /**
 +   * This function loops over all items in the top-level bin (covering the input sentence from
 +   * <s> ... </s>), looking for items with the goal LHS. For each of these, 
 +   * add all the items with GOAL_SYM state into the goal bin the goal bin has only one Item, which
 +   * itself has many hyperedges only "goal bin" should call this function
 +   */
 +  // note that the input bin is bin[0][n], not the goal bin
 +  boolean transitToGoal(Cell bin, List<FeatureFunction> featureFunctions, int sentenceLength) {
-     this.sortedNodes = new ArrayList<HGNode>();
++    this.sortedNodes = new ArrayList<>();
 +    HGNode goalItem = null;
 +
 +    for (HGNode antNode : bin.getSortedNodes()) {
 +      if (antNode.lhs == this.goalSymbol) {
 +        float logP = antNode.bestHyperedge.getBestDerivationScore();
-         List<HGNode> antNodes = new ArrayList<HGNode>();
++        List<HGNode> antNodes = new ArrayList<>();
 +        antNodes.add(antNode);
 +
 +        float finalTransitionLogP = ComputeNodeResult.computeFinalCost(featureFunctions, antNodes,
 +            0, sentenceLength, null, this.chart.getSentence());
 +
-         List<HGNode> previousItems = new ArrayList<HGNode>();
++        List<HGNode> previousItems = new ArrayList<>();
 +        previousItems.add(antNode);
 +
 +        HyperEdge dt = new HyperEdge(null, logP + finalTransitionLogP, finalTransitionLogP,
 +            previousItems, null);
 +
 +        if (null == goalItem) {
 +          goalItem = new HGNode(0, sentenceLength + 1, this.goalSymbol, null, dt, logP
 +              + finalTransitionLogP);
 +          this.sortedNodes.add(goalItem);
 +        } else {
 +          goalItem.addHyperedgeInNode(dt);
 +        }
 +      } // End if item.lhs == this.goalSymID
 +    } // End foreach Item in bin.get_sorted_items()
 +
 +    int itemsInGoalBin = getSortedNodes().size();
 +    if (1 != itemsInGoalBin) {
 +      LOG.error("the goal_bin does not have exactly one item");
 +      return false;
 +    }
 +
 +    return true;
 +  }
 +
 +  /**
 +   * a note about pruning: when a hyperedge gets created, it first needs to pass through
 +   * shouldPruneEdge filter. Then, if it does not trigger a new node (i.e. will be merged to an old
 +   * node), then does not trigger pruningNodes. If it does trigger a new node (either because its
 +   * signature is new or because its logP is better than the old node's logP), then it will trigger
 +   * pruningNodes, which might causes *other* nodes got pruned as well
 +   * */
 +
 +  /**
 +   * Creates a new hyperedge and adds it to the chart, subject to pruning. The logic of this
 +   * function is as follows: if the pruner permits the edge to be added, we build the new edge,
 +   * which ends in an HGNode. If this is the first time we've built an HGNode for this point in the
 +   * graph, it gets added automatically. Otherwise, we add the hyperedge to the existing HGNode,
 +   * possibly updating the HGNode's cache of the best incoming hyperedge.
 +   * 
 +   * @return the new hypernode, or null if the cell was pruned.
 +   */
 +  HGNode addHyperEdgeInCell(ComputeNodeResult result, Rule rule, int i, int j, List<HGNode> ants,
 +      SourcePath srcPath, boolean noPrune) {
 +
 +//    System.err.println(String.format("ADD_EDGE(%d-%d): %s", i, j, rule.getRuleString()));
 +//    if (ants != null) {
 +//      for (int xi = 0; xi < ants.size(); xi++) {
 +//        System.err.println(String.format("  -> TAIL %s", ants.get(xi)));
 +//      }
 +//    }
 +
 +    List<DPState> dpStates = result.getDPStates();
 +    float pruningEstimate = result.getPruningEstimate();
 +    float transitionLogP = result.getTransitionCost();
 +    float finalizedTotalLogP = result.getViterbiCost();
 +
 +    /**
 +     * Here, the edge has passed pre-pruning. The edge will be added to the chart in one of three
 +     * ways:
 +     * 
 +     * 1. If there is no existing node, a new one gets created and the edge is its only incoming
 +     * hyperedge.
 +     * 
 +     * 2. If there is an existing node, the edge will be added to its list of incoming hyperedges,
 +     * possibly taking place as the best incoming hyperedge for that node.
 +     */
 +
 +    HyperEdge hyperEdge = new HyperEdge(rule, finalizedTotalLogP, transitionLogP, ants, srcPath);
 +    HGNode newNode = new HGNode(i, j, rule.getLHS(), dpStates, hyperEdge, pruningEstimate);
 +
 +    /**
 +     * each node has a list of hyperedges, need to check whether the node is already exist, if
 +     * yes, just add the hyperedges, this may change the best logP of the node
 +     * */
 +    HGNode oldNode = this.nodesSigTbl.get(newNode.signature());
 +    if (null != oldNode) { // have an item with same states, combine items
 +      this.chart.nMerged++;
 +
 +      /**
 +       * the position of oldItem in this.heapItems may change, basically, we should remove the
 +       * oldItem, and re-insert it (linear time), this is too expense)
 +       **/
 +      if (newNode.getScore() > oldNode.getScore()) { // merge old to new: semiring plus
 +
 +        newNode.addHyperedgesInNode(oldNode.hyperedges);
 +        // This will update the HashMap, so that the oldNode is destroyed.
 +        addNewNode(newNode);
 +      } else {// merge new to old, does not trigger pruningItems
 +        oldNode.addHyperedgesInNode(newNode.hyperedges);
 +      }
 +
 +    } else { // first time item
 +      this.chart.nAdded++; // however, this item may not be used in the future due to pruning in
 +      // the hyper-graph
 +      addNewNode(newNode);
 +    }
 +
 +    return newNode;
 +  }
 +
 +  List<HGNode> getSortedNodes() {
 +    ensureSorted();
 +    return this.sortedNodes;
 +  }
 +  
 +  Map<Integer, SuperNode> getSortedSuperItems() {
 +    ensureSorted();
 +    return this.superNodesTbl;
 +  }
 +  
 +  // ===============================================================
 +  // Private Methods
 +  // ===============================================================
 +
 +  /**
 +   * two cases this function gets called (1) a new hyperedge leads to a non-existing node signature
 +   * (2) a new hyperedge's signature matches an old node's signature, but the best-logp of old node
 +   * is worse than the new hyperedge's logP
 +   * */
 +  private void addNewNode(HGNode node) {
 +    this.nodesSigTbl.put(node.signature(), node); // add/replace the item
 +    this.sortedNodes = null; // reset the list
 +    
 +//    System.err.println(String.format("** NEW NODE %s %d %d", Vocabulary.word(node.lhs), node.i, node.j));
 +
 +    // since this.sortedItems == null, this is not necessary because we will always call
 +    // ensure_sorted to reconstruct the this.tableSuperItems
 +    // add a super-items if necessary
 +    SuperNode si = this.superNodesTbl.get(node.lhs);
 +    if (null == si) {
 +      si = new SuperNode(node.lhs);
 +      this.superNodesTbl.put(node.lhs, si);
 +    }
 +    si.nodes.add(node);// TODO what about the dead items?
 +  }
 +
 +  /**
 +   * get a sorted list of Nodes in the cell, and also make sure the list of node in any SuperItem is
 +   * sorted, this will be called only necessary, which means that the list is not always sorted,
 +   * mainly needed for goal_bin and cube-pruning
 +   */
 +  private void ensureSorted() {
 +    if (null == this.sortedNodes) {
 +      
 +      // get sortedNodes.
 +      this.sortedNodes = new ArrayList<>(this.nodesSigTbl.size());
 +      for (HGNode node : this.nodesSigTbl.values()) {
 +        this.sortedNodes.add(node);
 +      }
 +
 +      // sort the node in an decreasing-LogP order 
 +      this.sortedNodes.sort(HGNode.inverseLogPComparator);
 +
 +      // TODO: we cannot create new SuperItem here because the DotItem link to them.
 +      // Thus, we clear nodes from existing SuperNodes
 +      for (SuperNode superNode : this.superNodesTbl.values()) {
 +        superNode.nodes.clear();
 +      }
 +
 +      for (HGNode node : this.sortedNodes) {
 +        SuperNode superNode = this.superNodesTbl.get(node.lhs);
 +        checkNotNull(superNode, "Does not have super Item, have to exist");
 +        superNode.nodes.add(node);
 +      }
 +
 +      // Remove SuperNodes who may not contain any nodes anymore due to pruning
 +      for (Iterator<Entry<Integer, SuperNode>> it = this.superNodesTbl.entrySet().iterator(); it.hasNext(); ) {
 +        Entry<Integer, SuperNode> entry = it.next();
 +        if (entry.getValue().nodes.isEmpty()) {
 +          it.remove();
 +        }
 +      }
 +    }
 +  }
 +}