You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/23 22:17:56 UTC
[39/50] [abbrv] incubator-joshua git commit: Merge branch 'master'
into 7
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
index 9b07a04,0000000..a2975c5
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
@@@ -1,730 -1,0 +1,735 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static org.apache.joshua.util.Constants.TM_PREFIX;
+import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
+import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.fragmentlm.Tree;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.Regex;
+import org.apache.joshua.util.io.LineReader;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Configuration file for Joshua decoder.
+ *
+ * When adding new features to Joshua, any new configurable parameters should be added to this
+ * class.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class JoshuaConfiguration {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JoshuaConfiguration.class);
+
+ // whether to construct a StructuredTranslation object for each request instead of
+ // printing to stdout. Used when the Decoder is used from Java directly.
+ public Boolean use_structured_output = false;
+
+ // If set to true, Joshua will lowercase the input, creating an annotation that marks the
+ // original case
+ public boolean lowercase = false;
+
+ // If set to true, Joshua will recapitalize the output by projecting the case from aligned
+ // source-side words
+ public boolean project_case = false;
+
+ // List of grammar files to read
- public ArrayList<String> tms = new ArrayList<String>();
++ public ArrayList<String> tms = new ArrayList<>();
+
+ // A rule cache for commonly used tries to avoid excess object allocations
+ // Testing shows there's up to ~95% hit rate when cache size is 5000 Trie nodes.
- public Integer cachedRuleSize = new Integer(5000);
++ public Integer cachedRuleSize = 5000;
+
+ /*
+ * The file to read the weights from (part of the sparse features implementation). Weights can
+ * also just be listed in the main config file.
+ */
+ public String weights_file = "";
+ // Default symbols. The symbol here should be enclosed in square brackets.
+ public String default_non_terminal = FormatUtils.ensureNonTerminalBrackets("X");
+ public String goal_symbol = FormatUtils.ensureNonTerminalBrackets("GOAL");
+
+ /*
+ * A list of OOV symbols in the form
+ *
+ * [X1] weight [X2] weight [X3] weight ...
+ *
+ * where the [X] symbols are nonterminals and the weights are weights. For each OOV word w in the
+ * input sentence, Joshua will create rules of the form
+ *
+ * X1 -> w (weight)
+ *
+ * If this is empty, an unweighted default_non_terminal is used.
+ */
+ public class OOVItem implements Comparable<OOVItem> {
- public String label;
++ public final String label;
+
- public float weight;
++ public final float weight;
+
+ OOVItem(String l, float w) {
+ label = l;
+ weight = w;
+ }
+ @Override
+ public int compareTo(OOVItem other) {
+ if (weight > other.weight)
+ return -1;
+ else if (weight < other.weight)
+ return 1;
+ return 0;
+ }
+ }
+
+ public ArrayList<OOVItem> oovList = null;
+
+ /*
+ * Whether to segment OOVs into a lattice
+ */
+ public boolean segment_oovs = false;
+
+ /*
+ * Enable lattice decoding.
+ */
+ public boolean lattice_decoding = false;
+
+ /*
+ * If false, sorting of the complete grammar is done at load time. If true, grammar tries are not
+ * sorted till they are first accessed. Amortized sorting means you get your first translation
+ * much, much quicker (good for debugging), but that per-sentence decoding is a bit slower.
+ */
+ public boolean amortized_sorting = true;
+ // syntax-constrained decoding
+ public boolean constrain_parse = false;
+
+ public boolean use_pos_labels = false;
+
+ // oov-specific
+ public boolean true_oovs_only = false;
+
+ /* Dynamic sentence-level filtering. */
+ public boolean filter_grammar = false;
+
+ /* The cube pruning pop limit. Set to 0 for exhaustive pruning. */
+ public int pop_limit = 100;
+
+ /* Maximum sentence length. Sentences longer than this are truncated. */
+ public int maxlen = 200;
+
+ /*
+ * N-best configuration.
+ */
+ // Make sure output strings in the n-best list are unique.
+ public boolean use_unique_nbest = true;
+
+ /* Include the phrasal alignments in the output (not word-level alignmetns at the moment). */
+ public boolean include_align_index = false;
+
+ /* The number of hypotheses to output by default. */
+ public int topN = 1;
+
+ /**
+ * This string describes the format of each line of output from the decoder (i.e., the
+ * translations). The string can include arbitrary text and also variables. The following
+ * variables are available:
+ *
+ * <pre>
+ * - %i the 0-indexed sentence number
+ * - %e the source string %s the translated sentence
+ * - %S the translated sentence with some basic capitalization and denormalization
+ * - %t the synchronous derivation
+ * - %f the list of feature values (as name=value pairs)
+ * - %c the model cost
+ * - %w the weight vector
+ * - %a the alignments between source and target words (currently unimplemented)
+ * - %d a verbose, many-line version of the derivation
+ * </pre>
+ */
+ public String outputFormat = "%i ||| %s ||| %f ||| %c";
+
+ /* The number of decoding threads to use (-threads). */
+ public int num_parallel_decoders = 1;
+
+ /*
+ * When true, _OOV is appended to all words that are passed through (useful for something like
+ * transliteration on the target side
+ */
+ public boolean mark_oovs = false;
+
+ /* Enables synchronous parsing. */
+ public boolean parse = false; // perform synchronous parsing
+
+
+ /* A list of the feature functions. */
- public ArrayList<String> features = new ArrayList<String>();
++ public ArrayList<String> features = new ArrayList<>();
+
+ /* A list of weights found in the main config file (instead of in a separate weights file) */
- public ArrayList<String> weights = new ArrayList<String>();
++ public ArrayList<String> weights = new ArrayList<>();
+
+ /* Determines whether to expect JSON input or plain lines */
- public enum INPUT_TYPE { plain, json };
++ public enum INPUT_TYPE { plain, json }
++
+ public INPUT_TYPE input_type = INPUT_TYPE.plain;
+
+ /* Type of server. Not sure we need to keep the regular TCP one around. */
- public enum SERVER_TYPE { none, TCP, HTTP };
++ public enum SERVER_TYPE { none, TCP, HTTP }
++
+ public SERVER_TYPE server_type = SERVER_TYPE.TCP;
+
+ /* If set, Joshua will start a (multi-threaded, per "threads") TCP/IP server on this port. */
+ public int server_port = 0;
+
+ /*
+ * Whether to do forest rescoring. If set to true, the references are expected on STDIN along with
+ * the input sentences in the following format:
+ *
+ * input sentence ||| ||| reference1 ||| reference2 ...
+ *
+ * (The second field is reserved for the output sentence for alignment and forced decoding).
+ */
+
+ public boolean rescoreForest = false;
+ public float rescoreForestWeight = 10.0f;
+
+ /*
+ * Location of fragment mapping file, which maps flattened SCFG rules to their internal
+ * representation.
+ */
+ public String fragmentMapFile = null;
+
+ /*
+ * Whether to use soft syntactic constraint decoding /fuzzy matching, which allows that any
+ * nonterminal may be substituted for any other nonterminal (except for OOV and GOAL)
+ */
+ public boolean fuzzy_matching = false;
+
+ public static final String SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME = "fuzzy_matching";
+
+ /***
+ * Phrase-based decoding parameters.
+ */
+
+ /* The search algorithm: currently either "cky" or "stack" */
+ public String search_algorithm = "cky";
+
+ /* The distortion limit */
+ public int reordering_limit = 8;
+
+ /* The number of target sides considered for each source side (after sorting by model weight) */
+ public int num_translation_options = 20;
+
+ /* If true, decode using a dot chart (standard CKY+); if false, use the much more efficient
+ * version of Sennrich (SSST 2014)
+ */
+ public boolean use_dot_chart = true;
+
+ /* Moses compatibility */
+ public boolean moses = false;
+
+ /* If true, just print out the weights found in the config file, and exit. */
+ public boolean show_weights_and_quit = false;
+
+ /* Read input from a file (Moses compatible flag) */
+ public String input_file = null;
+
+ /* Write n-best output to this file */
+ public String n_best_file = null;
+
+ /* Whether to look at source side for special annotations */
+ public boolean source_annotations = false;
+
+ /* Weights overridden from the command line */
+ public String weight_overwrite = "";
+
+ /**
+ * This method resets the state of JoshuaConfiguration back to the state after initialization.
+ * This is useful when for example making different calls to the decoder within the same java
+ * program, which otherwise leads to potential errors due to inconsistent state as a result of
+ * loading the configuration multiple times without resetting etc.
+ *
+ * This leads to the insight that in fact it may be an even better idea to refactor the code and
+ * make JoshuaConfiguration an object that is is created and passed as an argument, rather than a
+ * shared static object. This is just a suggestion for the next step.
+ *
+ */
+ public void reset() {
+ LOG.info("Resetting the JoshuaConfiguration to its defaults ...");
+ LOG.info("\n\tResetting the StatefullFF global state index ...");
+ LOG.info("\n\t...done");
+ StatefulFF.resetGlobalStateIndex();
- tms = new ArrayList<String>();
++ tms = new ArrayList<>();
+ weights_file = "";
+ default_non_terminal = "[X]";
- oovList = new ArrayList<OOVItem>();
++ oovList = new ArrayList<>();
+ oovList.add(new OOVItem(default_non_terminal, 1.0f));
+ goal_symbol = "[GOAL]";
+ amortized_sorting = true;
+ constrain_parse = false;
+ use_pos_labels = false;
+ true_oovs_only = false;
+ filter_grammar = false;
+ pop_limit = 100;
+ maxlen = 200;
+ use_unique_nbest = false;
+ include_align_index = false;
+ topN = 1;
+ outputFormat = "%i ||| %s ||| %f ||| %c";
+ num_parallel_decoders = 1;
+ mark_oovs = false;
+ // oracleFile = null;
+ parse = false; // perform synchronous parsing
- features = new ArrayList<String>();
- weights = new ArrayList<String>();
++ features = new ArrayList<>();
++ weights = new ArrayList<>();
+ server_port = 0;
+
+ reordering_limit = 8;
+ num_translation_options = 20;
+ LOG.info("...done");
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ /**
+ * To process command-line options, we write them to a file that looks like the config file, and
+ * then call readConfigFile() on it. It would be more general to define a class that sits on a
+ * stream and knows how to chop it up, but this was quicker to implement.
+ *
+ * @param options string array of command line options
+ */
+ public void processCommandLineOptions(String[] options) {
+ try {
+ File tmpFile = File.createTempFile("options", null, null);
+ PrintWriter out = new PrintWriter(new FileWriter(tmpFile));
+
+ for (int i = 0; i < options.length; i++) {
+ String key = options[i].substring(1);
+ if (i + 1 == options.length || options[i + 1].startsWith("-")) {
+ // if this is the last item, or if the next item
+ // is another flag, then this is a boolean flag
+ out.println(key + " = true");
+
+ } else {
+ out.print(key + " =");
+ while (i + 1 < options.length && ! options[i + 1].startsWith("-")) {
+ out.print(String.format(" %s", options[i + 1]));
+ i++;
+ }
+ out.println();
+ }
+ }
+ out.close();
+
+// LOG.info("Parameters overridden from the command line:");
+ this.readConfigFile(tmpFile.getCanonicalPath());
+
+ tmpFile.delete();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void readConfigFile(String configFile) throws IOException {
+
+ LineReader configReader = new LineReader(configFile, false);
+ try {
+ for (String line : configReader) {
+ line = line.trim(); // .toLowerCase();
+
+ if (Regex.commentOrEmptyLine.matches(line))
+ continue;
+
+ /*
+ * There are two kinds of substantive (non-comment, non-blank) lines: parameters and feature
+ * values. Parameters match the pattern "key = value"; all other substantive lines are
+ * interpreted as features.
+ */
+
- if (line.indexOf("=") != -1) { // parameters; (not feature function)
++ if (line.contains("=")) { // parameters; (not feature function)
+ String[] fds = Regex.equalsWithSpaces.split(line, 2);
+ if (fds.length < 2) {
+ LOG.warn("skipping config file line '{}'", line);
+ continue;
+ }
+
+ String parameter = normalize_key(fds[0]);
+
+ if (parameter.equals(normalize_key("lm"))) {
+ /* This is deprecated. This support old LM lines of the form
+ *
+ * lm = berkeleylm 5 false false 100 lm.gz
+ *
+ * LMs are now loaded as general feature functions, so we transform that to either
+ *
+ * LanguageModel -lm_order 5 -lm_type berkeleylm -lm_file lm.gz
+ *
+ * If the line were state minimizing:
+ *
+ * lm = kenlm 5 true false 100 lm.gz
+ *
+ * StateMinimizingLanguageModel -lm_order 5 -lm_file lm.gz
+ */
+
+ String[] tokens = fds[1].split("\\s+");
+ if (tokens[2].equals("true"))
+ features.add(String.format("StateMinimizingLanguageModel -lm_type kenlm -lm_order %s -lm_file %s",
+ tokens[1], tokens[5]));
+ else
+ features.add(String.format("LanguageModel -lm_type %s -lm_order %s -lm_file %s",
+ tokens[0], tokens[1], tokens[5]));
+
+ } else if (parameter.equals(normalize_key(TM_PREFIX))) {
+ /* If found, convert old format:
+ * tm = TYPE OWNER MAXSPAN PATH
+ * to new format
+ * tm = TYPE -owner OWNER -maxspan MAXSPAN -path PATH
+ */
+ String tmLine = fds[1];
+
+ String[] tokens = fds[1].split("\\s+");
+ if (! tokens[1].startsWith("-")) { // old format
+ tmLine = String.format("%s -owner %s -maxspan %s -path %s", tokens[0], tokens[1], tokens[2], tokens[3]);
+ LOG.warn("Converting deprecated TM line from '{}' -> '{}'", fds[1], tmLine);
+ }
+ tms.add(tmLine);
+
+ } else if (parameter.equals("v")) {
+
+ // This is already handled in ArgsParser, skip it here, easier than removing it there
+
+ } else if (parameter.equals(normalize_key("parse"))) {
+ parse = Boolean.parseBoolean(fds[1]);
+ LOG.debug("parse: {}", parse);
+
+ } else if (parameter.equals(normalize_key("oov-list"))) {
+ if (new File(fds[1]).exists()) {
- oovList = new ArrayList<OOVItem>();
++ oovList = new ArrayList<>();
+ try {
+ File file = new File(fds[1]);
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ try {
+ String str = br.readLine();
+ while (str != null) {
+ String[] tokens = str.trim().split("\\s+");
+
+ oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[0]),
+ (float) Math.log(Float.parseFloat(tokens[1]))));
+
+ str = br.readLine();
+ }
+ br.close();
+ } catch(IOException e){
+ System.out.println(e);
+ }
+ } catch(IOException e){
+ System.out.println(e);
+ }
+ Collections.sort(oovList);
+
+ } else {
+ String[] tokens = fds[1].trim().split("\\s+");
+ if (tokens.length % 2 != 0) {
+ throw new RuntimeException(String.format("* FATAL: invalid format for '%s'", fds[0]));
+ }
- oovList = new ArrayList<OOVItem>();
++ oovList = new ArrayList<>();
+
+ for (int i = 0; i < tokens.length; i += 2)
+ oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[i]),
+ (float) Math.log(Float.parseFloat(tokens[i + 1]))));
+
+ Collections.sort(oovList);
+ }
+
+ } else if (parameter.equals(normalize_key("lattice-decoding"))) {
+ lattice_decoding = true;
+
+ } else if (parameter.equals(normalize_key("segment-oovs"))) {
+ segment_oovs = true;
+ lattice_decoding = true;
+
+ } else if (parameter.equals(normalize_key("default-non-terminal"))) {
+ default_non_terminal = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
+ LOG.debug("default_non_terminal: {}", default_non_terminal);
+
+ } else if (parameter.equals(normalize_key("goal-symbol"))) {
+ goal_symbol = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
+ LOG.debug("goalSymbol: {}", goal_symbol);
+
+ } else if (parameter.equals(normalize_key("weights-file"))) {
+ weights_file = fds[1];
+
+ } else if (parameter.equals(normalize_key("constrain_parse"))) {
+ constrain_parse = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("true_oovs_only"))) {
+ true_oovs_only = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("filter-grammar"))) {
+ filter_grammar = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("amortize"))) {
+ amortized_sorting = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("use_pos_labels"))) {
+ use_pos_labels = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("use_unique_nbest"))) {
+ use_unique_nbest = Boolean.valueOf(fds[1]);
+ LOG.debug("use_unique_nbest: {}", use_unique_nbest);
+
+ } else if (parameter.equals(normalize_key("output-format"))) {
+ outputFormat = fds[1];
+ LOG.debug("output-format: {}", outputFormat);
+
+ } else if (parameter.equals(normalize_key("include_align_index"))) {
+ include_align_index = Boolean.valueOf(fds[1]);
+ LOG.debug("include_align_index: {}", include_align_index);
+
+ } else if (parameter.equals(normalize_key("top_n"))) {
+ topN = Integer.parseInt(fds[1]);
+ LOG.debug("topN: {}", topN);
+
+ } else if (parameter.equals(normalize_key("num_parallel_decoders"))
+ || parameter.equals(normalize_key("threads"))) {
+ num_parallel_decoders = Integer.parseInt(fds[1]);
+ if (num_parallel_decoders <= 0) {
+ throw new IllegalArgumentException(
+ "Must specify a positive number for num_parallel_decoders");
+ }
+ LOG.debug("num_parallel_decoders: {}", num_parallel_decoders);
+
+ } else if (parameter.equals(normalize_key("mark_oovs"))) {
+ mark_oovs = Boolean.valueOf(fds[1]);
+ LOG.debug("mark_oovs: {}", mark_oovs);
+
+ } else if (parameter.equals(normalize_key("pop-limit"))) {
+ pop_limit = Integer.parseInt(fds[1]);
+ LOG.info("pop-limit: {}", pop_limit);
+
+ } else if (parameter.equals(normalize_key("input-type"))) {
- if (fds[1].equals("json")) {
++ switch (fds[1]) {
++ case "json":
+ input_type = INPUT_TYPE.json;
- } else if (fds[1].equals("plain")) {
++ break;
++ case "plain":
+ input_type = INPUT_TYPE.plain;
- } else {
- throw new RuntimeException(String.format("* FATAL: invalid server type '%s'", fds[1]));
++ break;
++ default:
++ throw new RuntimeException(
++ String.format("* FATAL: invalid server type '%s'", fds[1]));
+ }
+ LOG.info(" input-type: {}", input_type);
+
+ } else if (parameter.equals(normalize_key("server-type"))) {
+ if (fds[1].toLowerCase().equals("tcp"))
+ server_type = SERVER_TYPE.TCP;
+ else if (fds[1].toLowerCase().equals("http"))
+ server_type = SERVER_TYPE.HTTP;
+
+ LOG.info(" server-type: {}", server_type);
+
+ } else if (parameter.equals(normalize_key("server-port"))) {
+ server_port = Integer.parseInt(fds[1]);
+ LOG.info(" server-port: {}", server_port);
+
+ } else if (parameter.equals(normalize_key("rescore-forest"))) {
+ rescoreForest = true;
+ LOG.info(" rescore-forest: {}", rescoreForest);
+
+ } else if (parameter.equals(normalize_key("rescore-forest-weight"))) {
+ rescoreForestWeight = Float.parseFloat(fds[1]);
+ LOG.info(" rescore-forest-weight: {}", rescoreForestWeight);
+
+ } else if (parameter.equals(normalize_key("maxlen"))) {
+ // reset the maximum length
+ maxlen = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals("c") || parameter.equals("config")) {
+ // this was used to send in the config file, just ignore it
- ;
+
+ } else if (parameter.equals(normalize_key("feature-function"))) {
+ // add the feature to the list of features for later processing
+ features.add(fds[1]);
+
+ } else if (parameter.equals(normalize_key("maxlen"))) {
+ // add the feature to the list of features for later processing
+ maxlen = Integer.parseInt(fds[1]);
+
+ } else if (parameter
+ .equals(normalize_key(SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME))) {
+ fuzzy_matching = Boolean.parseBoolean(fds[1]);
+ LOG.debug("fuzzy_matching: {}", fuzzy_matching);
+
+ } else if (parameter.equals(normalize_key("fragment-map"))) {
+ fragmentMapFile = fds[1];
+ Tree.readMapping(fragmentMapFile);
+
+ /** PHRASE-BASED PARAMETERS **/
+ } else if (parameter.equals(normalize_key("search"))) {
+ search_algorithm = fds[1];
+
+ if (!search_algorithm.equals("cky") && !search_algorithm.equals("stack")) {
+ throw new RuntimeException(
+ "-search must be one of 'stack' (for phrase-based decoding) " +
+ "or 'cky' (for hierarchical / syntactic decoding)");
+ }
+
+ if (search_algorithm.equals("cky") && include_align_index) {
+ throw new RuntimeException(
+ "include_align_index is currently not supported with cky search");
+ }
+
+ } else if (parameter.equals(normalize_key("reordering-limit"))) {
+ reordering_limit = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals(normalize_key("num-translation-options"))) {
+ num_translation_options = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals(normalize_key("no-dot-chart"))) {
+ use_dot_chart = false;
+
+ } else if (parameter.equals(normalize_key("moses"))) {
+ moses = true; // triggers some Moses-specific compatibility options
+
+ } else if (parameter.equals(normalize_key("show-weights"))) {
+ show_weights_and_quit = true;
+
+ } else if (parameter.equals(normalize_key("n-best-list"))) {
+ // for Moses compatibility
+ String[] tokens = fds[1].split("\\s+");
+ n_best_file = tokens[0];
+ if (tokens.length > 1)
+ topN = Integer.parseInt(tokens[1]);
+
+ } else if (parameter.equals(normalize_key("input-file"))) {
+ // for Moses compatibility
+ input_file = fds[1];
+
+ } else if (parameter.equals(normalize_key("weight-file"))) {
+ // for Moses, ignore
+
+ } else if (parameter.equals(normalize_key("weight-overwrite"))) {
+ weight_overwrite = fds[1];
+
+ } else if (parameter.equals(normalize_key("source-annotations"))) {
+ // Check source sentence
+ source_annotations = true;
+
+ } else if (parameter.equals(normalize_key("cached-rules-size"))) {
+ // Check source sentence
+ cachedRuleSize = Integer.parseInt(fds[1]);
+ } else if (parameter.equals(normalize_key("lowercase"))) {
+ lowercase = true;
+
+ } else if (parameter.equals(normalize_key("project-case"))) {
+ project_case = true;
+
+ } else {
+
+ if (parameter.equals(normalize_key("use-sent-specific-tm"))
+ || parameter.equals(normalize_key("add-combined-cost"))
+ || parameter.equals(normalize_key("use-tree-nbest"))
+ || parameter.equals(normalize_key("use-kenlm"))
+ || parameter.equals(normalize_key("useCubePrune"))
+ || parameter.equals(normalize_key("useBeamAndThresholdPrune"))
+ || parameter.equals(normalize_key("regexp-grammar"))) {
+ LOG.warn("ignoring deprecated parameter '{}'", fds[0]);
+
+ } else {
+ throw new RuntimeException("FATAL: unknown configuration parameter '" + fds[0] + "'");
+ }
+ }
+
+ LOG.info(" {} = '{}'", normalize_key(fds[0]), fds[1]);
+
+ } else {
+ /*
+ * Lines that don't have an equals sign and are not blank lines, empty lines, or comments,
+ * are feature values, which can be present in this file
+ */
+
+ weights.add(line);
+ }
+ }
+ } finally {
+ configReader.close();
+ }
+ }
+
+ /**
+ * Checks for invalid variable configurations
+ */
+ public void sanityCheck() {
+ }
+
+ /**
+ * Sets the verbosity level to v (0: OFF; 1: INFO; 2: DEBUG).
+ *
+ * @param v the verbosity level (0, 1, or 2)
+ */
+ public void setVerbosity(int v) {
+ Decoder.VERBOSE = v;
+ switch (Decoder.VERBOSE) {
+ case 0:
+ LogManager.getRootLogger().setLevel(Level.OFF);
+ break;
+ case 1:
+ LogManager.getRootLogger().setLevel(Level.INFO);
+ break;
+ case 2:
+ LogManager.getRootLogger().setLevel(Level.DEBUG);
+ break;
+ }
+ }
+
+ /**
+ * Normalizes parameter names by removing underscores and hyphens and lowercasing. This defines
+ * equivalence classes on external use of parameter names, permitting arbitrary_under_scores and
+ * camelCasing in paramter names without forcing the user to memorize them all. Here are some
+ * examples of equivalent ways to refer to parameter names:
+ * <pre>
+ * {pop-limit, poplimit, PopLimit, popLimit, pop_lim_it} {lmfile, lm-file, LM-FILE, lm_file}
+ * </pre>
+ *
+ * @param text the string to be normalized
+ * @return normalized key
+ *
+ */
+ public static String normalize_key(String text) {
+ return text.replaceAll("[-_]", "").toLowerCase();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
index 9f63cad,0000000..d3913fb
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
@@@ -1,446 -1,0 +1,443 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Scanner;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.joshua.util.Ngram;
+import org.apache.joshua.util.Regex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this class implements: (1) nbest min risk (MBR) reranking using BLEU as a gain funtion.
+ * <p>
+ * This assume that the string is unique in the nbest list In Hiero, due to spurious ambiguity, a
+ * string may correspond to many possible derivations, and ideally the probability of a string
+ * should be the sum of all the derivataions leading to that string. But, in practice, one normally
+ * uses a Viterbi approximation: the probability of a string is its best derivation probability So,
+ * if one want to deal with spurious ambiguity, he/she should do that before calling this class
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class NbestMinRiskReranker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NbestMinRiskReranker.class);
+
+ // TODO: this functionality is not implemented yet; default is to produce 1best without any
+ // feature scores;
+ boolean produceRerankedNbest = false;
+
+ double scalingFactor = 1.0;
+
- static int bleuOrder = 4;
- static boolean doNgramClip = true;
++ static final int bleuOrder = 4;
++ static final boolean doNgramClip = true;
+
- static boolean useGoogleLinearCorpusGain = false;
++ static final boolean useGoogleLinearCorpusGain = false;
+
- final PriorityBlockingQueue<RankerResult> resultsQueue =
- new PriorityBlockingQueue<RankerResult>();
++ final PriorityBlockingQueue<RankerResult> resultsQueue = new PriorityBlockingQueue<>();
+
+ public NbestMinRiskReranker(boolean produceRerankedNbest, double scalingFactor) {
+ this.produceRerankedNbest = produceRerankedNbest;
+ this.scalingFactor = scalingFactor;
+ }
+
+
+ public String processOneSent(List<String> nbest, int sentID) {
+ LOG.info("Now process sentence {}", sentID);
+
+ // step-0: preprocess
+ // assumption: each hyp has a formate:
+ // "sent_id ||| hyp_itself ||| feature scores ||| linear-combination-of-feature-scores(this should be logP)"
+
+ /* Quit if you find an empty hypothesis. */
+ if (nbest.size() == 1) {
+ String[] fields = Regex.threeBarsWithSpace.split(nbest.get(0));
+ if (fields[1].equals("") || Regex.spaces.matches(fields[1])) {
+ LOG.warn("-> sentence is empty");
+ return "";
+ }
+ }
+
- List<String> hypsItself = new ArrayList<String>();
++ List<String> hypsItself = new ArrayList<>();
+ // ArrayList<String> l_feat_scores = new ArrayList<String>();
- List<Double> baselineScores = new ArrayList<Double>(); // linear combination of all baseline
++ List<Double> baselineScores = new ArrayList<>(); // linear combination of all baseline
+ // features
- List<HashMap<String, Integer>> ngramTbls = new ArrayList<HashMap<String, Integer>>();
- List<Integer> sentLens = new ArrayList<Integer>();
++ List<HashMap<String, Integer>> ngramTbls = new ArrayList<>();
++ List<Integer> sentLens = new ArrayList<>();
+
+ for (String hyp : nbest) {
+ String[] fds = Regex.threeBarsWithSpace.split(hyp);
+ int tSentID = Integer.parseInt(fds[0]);
+ if (sentID != tSentID) {
+ throw new RuntimeException("sentence_id does not match");
+ }
+ String hypothesis = (fds.length >= 4) ? fds[1] : "";
+ hypsItself.add(hypothesis);
+
+ String[] words = Regex.spaces.split(hypothesis);
+ sentLens.add(words.length);
+
- HashMap<String, Integer> ngramTbl = new HashMap<String, Integer>();
++ HashMap<String, Integer> ngramTbl = new HashMap<>();
+ Ngram.getNgrams(ngramTbl, 1, bleuOrder, words);
+ ngramTbls.add(ngramTbl);
+
+ // l_feat_scores.add(fds[2]);
+
+ // The value of finalIndex is expected to be 3,
+ // unless the hyp_itself is empty,
+ // in which case finalIndex will be 2.
+ int finalIndex = fds.length - 1;
+ baselineScores.add(Double.parseDouble(fds[finalIndex]));
+
+ }
+
+ // step-1: get normalized distribution
+
+ /**
+ * value in baselineScores will be changed to normalized probability
+ * */
+ computeNormalizedProbs(baselineScores, scalingFactor);
+
- List<Double> normalizedProbs = baselineScores;
-
+ // === required by google linear corpus gain
+ HashMap<String, Double> posteriorCountsTbl = null;
+ if (useGoogleLinearCorpusGain) {
- posteriorCountsTbl = new HashMap<String, Double>();
- getGooglePosteriorCounts(ngramTbls, normalizedProbs, posteriorCountsTbl);
++ posteriorCountsTbl = new HashMap<>();
++ getGooglePosteriorCounts(ngramTbls, baselineScores, posteriorCountsTbl);
+ }
+
+
+ // step-2: rerank the nbest
+ /**
+ * TODO: zhifei: now the re-ranking takes O(n^2) where n is the size of the nbest. But, we can
+ * significantly speed up this (leadding to O(n)) by first estimating a model on nbest, and then
+ * rerank the nbest using the estimated model.
+ * */
+ double bestGain = -1000000000;// set as worst gain
+ String bestHyp = null;
- List<Double> gains = new ArrayList<Double>();
++ List<Double> gains = new ArrayList<>();
+ for (int i = 0; i < hypsItself.size(); i++) {
+ String curHyp = hypsItself.get(i);
+ int curHypLen = sentLens.get(i);
+ HashMap<String, Integer> curHypNgramTbl = ngramTbls.get(i);
+ // double cur_gain = computeGain(cur_hyp, l_hyp_itself, l_normalized_probs);
+ double curGain = 0;
+ if (useGoogleLinearCorpusGain) {
+ curGain = computeExpectedLinearCorpusGain(curHypLen, curHypNgramTbl, posteriorCountsTbl);
+ } else {
+ curGain =
- computeExpectedGain(curHypLen, curHypNgramTbl, ngramTbls, sentLens, normalizedProbs);
++ computeExpectedGain(curHypLen, curHypNgramTbl, ngramTbls, sentLens, baselineScores);
+ }
+
+ gains.add(curGain);
+ if (i == 0 || curGain > bestGain) { // maximize
+ bestGain = curGain;
+ bestHyp = curHyp;
+ }
+ }
+
+ // step-3: output the 1best or nbest
+ if (this.produceRerankedNbest) {
+ // TOTO: sort the list and write the reranked nbest; Use Collections.sort(List list,
+ // Comparator c)
+ } else {
+ /*
+ * this.out.write(best_hyp); this.out.write("\n"); out.flush();
+ */
+ }
+
+ LOG.info("best gain: {}", bestGain);
+ if (null == bestHyp) {
+ throw new RuntimeException("mbr reranked one best is null, must be wrong");
+ }
+ return bestHyp;
+ }
+
+
+ /**
+ * based on a list of log-probabilities in nbestLogProbs, obtain a normalized distribution, and
+ * put the normalized probability (real value in [0,1]) into nbestLogProbs
+ *
+ * @param nbestLogProbs a {@link java.util.List} of {@link java.lang.Double} representing nbestLogProbs
+ * @param scalingFactor double value representing scaling factor
+ */
+ // get a normalized distributeion and put it back to nbestLogProbs
+ static public void computeNormalizedProbs(List<Double> nbestLogProbs, double scalingFactor) {
+
+ // === get noralization constant, remember features, remember the combined linear score
+ double normalizationConstant = Double.NEGATIVE_INFINITY;// log-semiring
+
+ for (double logp : nbestLogProbs) {
+ normalizationConstant = addInLogSemiring(normalizationConstant, logp * scalingFactor, 0);
+ }
+ // System.out.println("normalization_constant (logP) is " + normalization_constant);
+
+ // === get normalized prob for each hyp
+ double tSum = 0;
+ for (int i = 0; i < nbestLogProbs.size(); i++) {
+
+ double normalizedProb =
+ Math.exp(nbestLogProbs.get(i) * scalingFactor - normalizationConstant);
+ tSum += normalizedProb;
+ nbestLogProbs.set(i, normalizedProb);
+
+ if (Double.isNaN(normalizedProb)) {
+ throw new RuntimeException("prob is NaN, must be wrong\nnbest_logps.get(i): "
+ + nbestLogProbs.get(i) + "; scaling_factor: " + scalingFactor
+ + "; normalization_constant:" + normalizationConstant);
+ }
+ // logger.info("probability: " + normalized_prob);
+ }
+
+ // sanity check
+ if (Math.abs(tSum - 1.0) > 1e-4) {
+ throw new RuntimeException("probabilities not sum to one, must be wrong");
+ }
+
+ }
+
+
+ // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
+ // curHyp: e
+ // trueHyp: e'
+ public double computeExpectedGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
+ List<HashMap<String, Integer>> ngramTbls, List<Integer> sentLens, List<Double> nbestProbs) {
+
+ // ### get noralization constant, remember features, remember the combined linear score
+ double gain = 0;
+
+ for (int i = 0; i < nbestProbs.size(); i++) {
+ HashMap<String, Integer> trueHypNgramTbl = ngramTbls.get(i);
+ double trueProb = nbestProbs.get(i);
+ int trueLen = sentLens.get(i);
+ gain +=
+ trueProb
+ * BLEU.computeSentenceBleu(trueLen, trueHypNgramTbl, curHypLen, curHypNgramTbl,
+ doNgramClip, bleuOrder);
+ }
+ // System.out.println("Gain is " + gain);
+ return gain;
+ }
+
+ // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
+ // curHyp: e
+ // trueHyp: e'
+ static public double computeExpectedGain(String curHyp, List<String> nbestHyps,
+ List<Double> nbestProbs) {
+ // ### get noralization constant, remember features, remember the combined linear score
+ double gain = 0;
+
+ for (int i = 0; i < nbestHyps.size(); i++) {
+ String trueHyp = nbestHyps.get(i);
+ double trueProb = nbestProbs.get(i);
+ gain += trueProb * BLEU.computeSentenceBleu(trueHyp, curHyp, doNgramClip, bleuOrder);
+ }
+ // System.out.println("Gain is " + gain);
+ return gain;
+ }
+
+ void getGooglePosteriorCounts(List<HashMap<String, Integer>> ngramTbls,
+ List<Double> normalizedProbs, HashMap<String, Double> posteriorCountsTbl) {
+ // TODO
+ }
+
+ double computeExpectedLinearCorpusGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
+ HashMap<String, Double> posteriorCountsTbl) {
+ // TODO
+ double[] thetas = {-1, 1, 1, 1, 1};
+
+ double res = 0;
+ res += thetas[0] * curHypLen;
+ for (Entry<String, Integer> entry : curHypNgramTbl.entrySet()) {
+ String key = entry.getKey();
+ String[] tem = Regex.spaces.split(key);
+
+ double post_prob = posteriorCountsTbl.get(key);
+ res += entry.getValue() * post_prob * thetas[tem.length];
+ }
+ return res;
+ }
+
+ // OR: return Math.log(Math.exp(x) + Math.exp(y));
+ static private double addInLogSemiring(double x, double y, int addMode) {// prevent over-flow
+ if (addMode == 0) { // sum
+ if (x == Double.NEGATIVE_INFINITY) {// if y is also n-infinity, then return n-infinity
+ return y;
+ }
+ if (y == Double.NEGATIVE_INFINITY) {
+ return x;
+ }
+
+ if (y <= x) {
+ return x + Math.log(1 + Math.exp(y - x));
+ } else {
+ return y + Math.log(1 + Math.exp(x - y));
+ }
+ } else if (addMode == 1) { // viter-min
+ return (x <= y) ? x : y;
+ } else if (addMode == 2) { // viter-max
+ return (x >= y) ? x : y;
+ } else {
+ throw new RuntimeException("invalid add mode");
+ }
+ }
+
+
+
+ public static void main(String[] args) throws IOException {
+
+ // If you don't know what to use for scaling factor, try using 1
+
+ if (args.length < 2) {
+ String msg = "usage: java NbestMinRiskReranker <produce_reranked_nbest> <scaling_factor> "
+ + "[numThreads]";
+ System.err.println(msg);
+ LOG.error(msg);
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ boolean produceRerankedNbest = Boolean.valueOf(args[0].trim());
+ double scalingFactor = Double.parseDouble(args[1].trim());
+ int numThreads = (args.length > 2) ? Integer.parseInt(args[2].trim()) : 1;
+
+
+ NbestMinRiskReranker mbrReranker =
+ new NbestMinRiskReranker(produceRerankedNbest, scalingFactor);
+
+ LOG.info("Running mbr reranking");
+
+ int oldSentID = -1;
- List<String> nbest = new ArrayList<String>();
++ List<String> nbest = new ArrayList<>();
+
+ Scanner scanner = new Scanner(System.in, "UTF-8");
+
+ if (numThreads == 1) {
+
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ String[] fds = Regex.threeBarsWithSpace.split(line);
+ int newSentID = Integer.parseInt(fds[0]);
+ if (oldSentID != -1 && oldSentID != newSentID) {
+ if (nbest.size() > 0) {
+ String best_hyp = mbrReranker.processOneSent(nbest, oldSentID);// nbest: list of unique
+ // strings
+ System.out.println(best_hyp);
+ } else {
+ System.out.println();
+ }
+ nbest.clear();
+ }
+ oldSentID = newSentID;
+ if (!fds[1].matches("^\\s*$")) nbest.add(line);
+ }
+
+ // last nbest
+ if (oldSentID >= 0) {
+ String bestHyp = mbrReranker.processOneSent(nbest, oldSentID);
+ System.out.println(bestHyp);
+ nbest.clear();
+ }
+
+ } else {
+
+ ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
+
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ String[] fds = Regex.threeBarsWithSpace.split(line);
+ int newSentID = Integer.parseInt(fds[0]);
+ if (oldSentID != -1 && oldSentID != newSentID) {
+
+ threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
+
+ nbest.clear();
+ }
+ oldSentID = newSentID;
+ nbest.add(line);
+ }
+
+ // last nbest
+ threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
+ nbest.clear();
+
+ threadPool.shutdown();
+
+ try {
+ threadPool.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+
+ while (!mbrReranker.resultsQueue.isEmpty()) {
+ RankerResult result = mbrReranker.resultsQueue.remove();
+ String best_hyp = result.toString();
+ System.out.println(best_hyp);
+ }
+ } catch (InterruptedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ scanner.close();
+
+ LOG.info("Total running time (seconds) is {} ",
+ (System.currentTimeMillis() - startTime) / 1000.0);
+ }
+
+ private class RankerTask implements Runnable {
+
+ final List<String> nbest;
+ final int sentID;
+
+ RankerTask(final List<String> nbest, final int sentID) {
- this.nbest = new ArrayList<String>(nbest);
++ this.nbest = new ArrayList<>(nbest);
+ this.sentID = sentID;
+ }
+
+ public void run() {
+ String result = processOneSent(nbest, sentID);
+ resultsQueue.add(new RankerResult(result, sentID));
+ }
+
+ }
+
+ private static class RankerResult implements Comparable<RankerResult> {
+ final String result;
+ final Integer sentenceNumber;
+
+ RankerResult(String result, int sentenceNumber) {
+ this.result = result;
+ this.sentenceNumber = sentenceNumber;
+ }
+
+ public int compareTo(RankerResult o) {
+ return sentenceNumber.compareTo(o.sentenceNumber);
+ }
+
+ public String toString() {
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
index 0e167d1,0000000..9f32d31
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
@@@ -1,154 -1,0 +1,154 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.List;
+import java.util.Map;
+
++import org.apache.joshua.decoder.io.DeNormalize;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * A StructuredTranslation instance provides a more structured access to
+ * translation results than the string-based Translation class.
+ * This is useful if the decoder is encapsulated in a larger project, instead
+ * of simply writing to a file or stdout.
+ * StructuredTranslation encodes all relevant information about a derivation,
+ * namely output string, tokens, score, features, and word alignment.
+ *
+ * @author fhieber
+ */
+public class StructuredTranslation {
+
+ private final Sentence sourceSentence;
+ private final String translationString;
+ private final List<String> translationTokens;
+ private final float translationScore;
+ private final List<List<Integer>> translationWordAlignments;
+ private final Map<String,Float> translationFeatures;
+ private final float extractionTime;
+
+ public StructuredTranslation(
+ final Sentence sourceSentence,
+ final String translationString,
+ final List<String> translationTokens,
+ final float translationScore,
+ final List<List<Integer>> translationWordAlignments,
+ final Map<String,Float> translationFeatures,
+ final float extractionTime) {
+ this.sourceSentence = sourceSentence;
+ this.translationString = translationString;
+ this.translationTokens = translationTokens;
+ this.translationScore = translationScore;
+ this.translationWordAlignments = translationWordAlignments;
+ this.translationFeatures = translationFeatures;
+ this.extractionTime = extractionTime;
+ }
+
+ public Sentence getSourceSentence() {
+ return sourceSentence;
+ }
+
+ public int getSentenceId() {
+ return sourceSentence.id();
+ }
+
+ /**
+ * Produces the raw translation hypothesis (still tokenized).
+ *
+ * @return the raw translation hypothesis
+ */
+ public String getTranslationString() {
+ return translationString;
+ }
+
+ /**
+ * Returns the output string formatted according to JoshuaConfiguration.output_format.
+ *
+ * @return the formatted string
+ */
+ public String getFormattedTranslationString() {
- throw new RuntimeException("Not yet implemented");
++ return DeNormalize.processSingleLine(maybeProjectCase(getTranslationString()));
+ }
+
+ public List<String> getTranslationTokens() {
+ return translationTokens;
+ }
+
+ public float getTranslationScore() {
+ return translationScore;
+ }
+
+ /**
+ * Returns a list of target to source alignments.
+ *
+ * @return a list of target to source alignments
+ */
+ public List<List<Integer>> getTranslationWordAlignments() {
+ return translationWordAlignments;
+ }
+
+ public Map<String,Float> getTranslationFeatures() {
+ return translationFeatures;
+ }
+
+ /**
+ * Time taken to build output information from the hypergraph.
+ * @return the time taken to build output information from the hypergraph
+ */
+ public Float getExtractionTime() {
+ return extractionTime;
+ }
+
+ /**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
- * @param hypothesis todo
- * @param state todo
- * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
++ * @param hypothesis the string hypothesis
++ * @return source-side lettercase to target, and appends the alignment from to the source-side sentence
+ */
+ private String maybeProjectCase(String hypothesis) {
+ String output = hypothesis;
+
+ JoshuaConfiguration config = sourceSentence.config;
+ if (config.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = getTranslationWordAlignments();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sourceSentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
index ce4f32c,0000000..6453bd1
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
@@@ -1,116 -1,0 +1,116 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.segment_file.Sentence;
++import org.apache.joshua.decoder.segment_file.Token;
++import org.apache.joshua.util.FormatUtils;
+
+/**
+ * This factory provides methods to create StructuredTranslation objects
+ * from either Viterbi derivations or KBest derivations.
+ *
+ * @author fhieber
+ */
+public class StructuredTranslationFactory {
+
+ /**
+ * Returns a StructuredTranslation instance from the Viterbi derivation.
+ *
+ * @param sourceSentence the source sentence
+ * @param hypergraph the hypergraph object
+ * @param featureFunctions the list of active feature functions
+ * @return A StructuredTranslation object representing the Viterbi derivation.
+ */
+ public static StructuredTranslation fromViterbiDerivation(
+ final Sentence sourceSentence,
+ final HyperGraph hypergraph,
+ final List<FeatureFunction> featureFunctions) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(getViterbiString(hypergraph));
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ extractTranslationScore(hypergraph),
+ getViterbiWordAlignmentList(hypergraph),
+ getViterbiFeatures(hypergraph, featureFunctions, sourceSentence).toStringMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation from an empty decoder output
+ * @param sourceSentence the source sentence
+ * @return a StructuredTranslation object
+ */
+ public static StructuredTranslation fromEmptyOutput(final Sentence sourceSentence) {
+ return new StructuredTranslation(
+ sourceSentence, "", emptyList(), 0, emptyList(), emptyMap(), 0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation instance from a KBest DerivationState.
+ * @param sourceSentence Sentence object representing the source.
+ * @param derivationState the KBest DerivationState.
+ * @return A StructuredTranslation object representing the derivation encoded by derivationState.
+ */
+ public static StructuredTranslation fromKBestDerivation(
+ final Sentence sourceSentence,
+ final DerivationState derivationState) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(derivationState.getHypothesis());
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ derivationState.getModelCost(),
+ derivationState.getWordAlignmentList(),
+ derivationState.getFeatures().toStringMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ private static float extractTranslationScore(final HyperGraph hypergraph) {
+ if (hypergraph == null) {
+ return 0;
+ } else {
+ return hypergraph.goalNode.getScore();
+ }
+ }
+
+ private static List<String> extractTranslationTokens(final String translationString) {
+ if (translationString.isEmpty()) {
+ return emptyList();
+ } else {
+ return asList(translationString.split("\\s+"));
+ }
+ }
-
-
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
index cdda63e,0000000..260b0ac
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
@@@ -1,237 -1,0 +1,239 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static java.util.Arrays.asList;
+import static org.apache.joshua.decoder.StructuredTranslationFactory.fromViterbiDerivation;
+import static org.apache.joshua.decoder.ff.FeatureMap.hashFeature;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignments;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.StringWriter;
++import java.util.Collections;
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.lm.StateMinimizingLanguageModel;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor;
+import org.apache.joshua.decoder.io.DeNormalize;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents translated input objects (sentences or lattices). It is aware of the source
+ * sentence and id and contains the decoded hypergraph. Translation objects are returned by
+ * DecoderThread instances to the InputHandler, where they are assembled in order for output.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Felix Hieber fhieber@amazon.com
+ */
+
+public class Translation {
+ private static final Logger LOG = LoggerFactory.getLogger(Translation.class);
- private Sentence source;
++ private final Sentence source;
+
+ /**
+ * This stores the output of the translation so we don't have to hold onto the hypergraph while we
+ * wait for the outputs to be assembled.
+ */
+ private String output = null;
+
+ /**
+ * Stores the list of StructuredTranslations.
+ * If joshuaConfig.topN == 0, will only contain the Viterbi translation.
+ * Else it will use KBestExtractor to populate this list.
+ */
+ private List<StructuredTranslation> structuredTranslations = null;
+
+ public Translation(Sentence source, HyperGraph hypergraph,
+ List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) {
+ this.source = source;
+
+ /**
+ * Structured output from Joshua provides a way to programmatically access translation results
+ * from downstream applications, instead of writing results as strings to an output buffer.
+ */
+ if (joshuaConfiguration.use_structured_output) {
+
+ if (joshuaConfiguration.topN == 0) {
+ /*
+ * Obtain Viterbi StructuredTranslation
+ */
+ StructuredTranslation translation = fromViterbiDerivation(source, hypergraph, featureFunctions);
+ this.output = translation.getTranslationString();
- structuredTranslations = asList(translation);
++ structuredTranslations = Collections.singletonList(translation);
+
+ } else {
+ /*
+ * Get K-Best list of StructuredTranslations
+ */
+ final KBestExtractor kBestExtractor = new KBestExtractor(source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
+ structuredTranslations = kBestExtractor.KbestExtractOnHG(hypergraph, joshuaConfiguration.topN);
+ if (structuredTranslations.isEmpty()) {
- structuredTranslations = asList(StructuredTranslationFactory.fromEmptyOutput(source));
++ structuredTranslations = Collections
++ .singletonList(StructuredTranslationFactory.fromEmptyOutput(source));
+ this.output = "";
+ } else {
+ this.output = structuredTranslations.get(0).getTranslationString();
+ }
+ // TODO: We omit the BLEU rescoring for now since it is not clear whether it works at all and what the desired output is below.
+ }
+
+ } else {
+
+ StringWriter sw = new StringWriter();
+ BufferedWriter out = new BufferedWriter(sw);
+
+ try {
+
+ if (hypergraph != null) {
+
+ long startTime = System.currentTimeMillis();
+
+ if (joshuaConfiguration.topN == 0) {
+
+ /* construct Viterbi output */
+ final String best = getViterbiString(hypergraph);
+
+ LOG.info("Translation {}: {} {}", source.id(), hypergraph.goalNode.getScore(), best);
+
+ /*
+ * Setting topN to 0 turns off k-best extraction, in which case we need to parse through
+ * the output-string, with the understanding that we can only substitute variables for the
+ * output string, sentence number, and model score.
+ */
+ String translation = joshuaConfiguration.outputFormat
+ .replace("%s", removeSentenceMarkers(best))
+ .replace("%S", DeNormalize.processSingleLine(best))
+ .replace("%c", String.format("%.3f", hypergraph.goalNode.getScore()))
+ .replace("%i", String.format("%d", source.id()));
+
+ if (joshuaConfiguration.outputFormat.contains("%a")) {
+ translation = translation.replace("%a", getViterbiWordAlignments(hypergraph));
+ }
+
+ if (joshuaConfiguration.outputFormat.contains("%f")) {
+ final FeatureVector features = getViterbiFeatures(hypergraph, featureFunctions, source);
+ translation = translation.replace("%f", features.textFormat());
+ }
+
+ out.write(translation);
+ out.newLine();
+
+ } else {
+
+ final KBestExtractor kBestExtractor = new KBestExtractor(
+ source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+
+ if (joshuaConfiguration.rescoreForest) {
+ final int bleuFeatureHash = hashFeature("BLEU");
+ Decoder.weights.add(bleuFeatureHash, joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+
+ Decoder.weights.add(bleuFeatureHash, -joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+ }
+ }
+
+ float seconds = (float) (System.currentTimeMillis() - startTime) / 1000.0f;
+ LOG.info("Input {}: {}-best extraction took {} seconds", id(),
+ joshuaConfiguration.topN, seconds);
+
+ } else {
+
+ // Failed translations and blank lines get empty formatted outputs
+ out.write(getFailedTranslationOutput(source, joshuaConfiguration));
+ out.newLine();
+
+ }
+
+ out.flush();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ this.output = sw.toString();
+
+ }
+
+ // remove state from StateMinimizingLanguageModel instances in features.
+ destroyKenLMStates(featureFunctions);
+
+ }
+
+ public Sentence getSourceSentence() {
+ return this.source;
+ }
+
+ public int id() {
+ return source.id();
+ }
+
+ @Override
+ public String toString() {
+ return output;
+ }
+
+ private String getFailedTranslationOutput(final Sentence source, final JoshuaConfiguration joshuaConfiguration) {
+ return joshuaConfiguration.outputFormat
+ .replace("%s", source.source())
+ .replace("%e", "")
+ .replace("%S", "")
+ .replace("%t", "()")
+ .replace("%i", Integer.toString(source.id()))
+ .replace("%f", "")
+ .replace("%c", "0.000");
+ }
+
+ /**
+ * Returns the StructuredTranslations
+ * if JoshuaConfiguration.use_structured_output == True.
+ * @throws RuntimeException if JoshuaConfiguration.use_structured_output == False.
+ * @return List of StructuredTranslations.
+ */
+ public List<StructuredTranslation> getStructuredTranslations() {
+ if (structuredTranslations == null) {
+ throw new RuntimeException(
+ "No StructuredTranslation objects created. You should set JoshuaConfigration.use_structured_output = true");
+ }
+ return structuredTranslations;
+ }
+
+ /**
+ * KenLM hack. If using KenLMFF, we need to tell KenLM to delete the pool used to create chart
+ * objects for this sentence.
+ */
+ private void destroyKenLMStates(final List<FeatureFunction> featureFunctions) {
+ for (FeatureFunction feature : featureFunctions) {
+ if (feature instanceof StateMinimizingLanguageModel) {
+ ((StateMinimizingLanguageModel) feature).destroyPool(getSourceSentence().id());
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
index e607225,0000000..1eb859a
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
@@@ -1,158 -1,0 +1,158 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+
+/**
+ * This class represents a streaming sequence of translations. It is returned by the main entry
+ * point to the Decoder object, the call to decodeAll. The translations here are parallel to the
+ * input sentences in the corresponding TranslationRequest object. Because of parallelization, the
+ * translated sentences might be computed out of order. Each Translation is sent to this
+ * Translations object by a DecoderThreadRunner via the record() function, which places the
+ * Translation in the right place. When the next translation in a sequence is available, next() is
+ * notified.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Translations implements Iterator<Translation>, Iterable<Translation> {
+
+ /* The source sentences to be translated. */
+ private TranslationRequestStream request = null;
+
+ /*
+ * This records the index of the sentence at the head of the underlying list. The iterator's
+ * next() blocks when the value at this position in the translations LinkedList is null.
+ */
+ private int currentID = 0;
+
+ /* The set of translated sentences. */
+ private LinkedList<Translation> translations = null;
+
+ private boolean spent = false;
+
+ private Translation nextTranslation;
+
+ public Translations(TranslationRequestStream request) {
+ this.request = request;
- this.translations = new LinkedList<Translation>();
++ this.translations = new LinkedList<>();
+ }
+
+ /**
+ * This is called when null is received from the TranslationRequest, indicating that there are no
+ * more input sentences to translated. That in turn means that the request size will no longer
+ * grow. We then notify any waiting thread if the last ID we've processed is the last one, period.
+ */
+ public void finish() {
+ synchronized (this) {
+ spent = true;
+ if (currentID == request.size()) {
+ this.notifyAll();
+ }
+ }
+ }
+
+ /**
+ * This is called whenever a translation is completed by one of the decoder threads. There may be
+ * a current output thread waiting for the current translation, which is determined by checking if
+ * the ID of the translation is the same as the one being waited for (currentID). If so, the
+ * thread waiting for it is notified.
+ *
+ * @param translation a translated input object
+ */
+ public void record(Translation translation) {
+ synchronized (this) {
+
+ /* Pad the set of translations with nulls to accommodate the new translation. */
+ int offset = translation.id() - currentID;
+ while (offset >= translations.size())
+ translations.add(null);
+ translations.set(offset, translation);
+
+ /*
+ * If the id of the current translation is at the head of the list (first element), then we
+ * have the next Translation to be return, and we should notify anyone waiting on next(),
+ * which will then remove the item and increment the currentID.
+ */
+ if (translation.id() == currentID) {
+ this.notify();
+ }
+ }
+ }
+
+ /**
+ * Returns the next Translation, blocking if necessary until it's available, since the next
+ * Translation might not have been produced yet.
+ *
+ * @return first element from the list of {@link org.apache.joshua.decoder.Translation}'s
+ */
+ @Override
+ public Translation next() {
+ synchronized(this) {
+ if (this.hasNext()) {
+ Translation t = this.nextTranslation;
+ this.nextTranslation = null;
+ return t;
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ synchronized (this) {
+
+ if (nextTranslation != null)
+ return true;
+
+ /*
+ * If there are no more input sentences, and we've already distributed what we then know is
+ * the last one, we're done.
+ */
+ if (spent && currentID == request.size())
+ return false;
+
+ /*
+ * Otherwise, there is another sentence. If it's not available already, we need to wait for
+ * it.
+ */
+ if (translations.size() == 0 || translations.peek() == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ /* We now have the sentence and can return it. */
+ currentID++;
+ this.nextTranslation = translations.poll();
+ return this.nextTranslation != null;
+ }
+ }
+
+ @Override
+ public Iterator<Translation> iterator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc756709/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
index 10b9200,0000000..cfcd06b
mode 100644,000000..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
@@@ -1,294 -1,0 +1,294 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this class implement functions: (1) combine small itesm into larger ones using rules, and create
+ * items and hyper-edges to construct a hyper-graph, (2) evaluate model score for items, (3)
+ * cube-pruning Note: Bin creates Items, but not all Items will be used in the hyper-graph
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Zhifei Li, <zh...@gmail.com>
+ */
+class Cell {
+
+ // ===============================================================
+ // Static fields
+ // ===============================================================
+ private static final Logger LOG = LoggerFactory.getLogger(Cell.class);
+
+
+ // The chart this cell belongs to
+ private Chart chart = null;
+
+ // The top-level (goal) symbol
- private int goalSymbol;
++ private final int goalSymbol;
+
+ // to maintain uniqueness of nodes
- private HashMap<HGNode.Signature, HGNode> nodesSigTbl = new LinkedHashMap<HGNode.Signature, HGNode>();
++ private final HashMap<HGNode.Signature, HGNode> nodesSigTbl = new LinkedHashMap<>();
+
+ // signature by lhs
- private Map<Integer, SuperNode> superNodesTbl = new HashMap<Integer, SuperNode>();
++ private final Map<Integer, SuperNode> superNodesTbl = new HashMap<>();
+
+ /**
+ * sort values in nodesSigTbl, we need this list when necessary
+ */
+ private List<HGNode> sortedNodes = null;
+
+
+ // ===============================================================
+ // Constructor
+ // ===============================================================
+
+ public Cell(Chart chart, int goalSymID) {
+ this.chart = chart;
+ this.goalSymbol = goalSymID;
+ }
+
+ public Cell(Chart chart, int goal_sym_id, int constraint_symbol_id) {
+ this(chart, goal_sym_id);
+ }
+
+ // ===============================================================
+ // Package-protected methods
+ // ===============================================================
+
+ public Set<Integer> getKeySet() {
+ return superNodesTbl.keySet();
+ }
+
+ public SuperNode getSuperNode(int lhs) {
+ return superNodesTbl.get(lhs);
+ }
+
+ /**
+ * This function loops over all items in the top-level bin (covering the input sentence from
+ * <s> ... </s>), looking for items with the goal LHS. For each of these,
+ * add all the items with GOAL_SYM state into the goal bin the goal bin has only one Item, which
+ * itself has many hyperedges only "goal bin" should call this function
+ */
+ // note that the input bin is bin[0][n], not the goal bin
+ boolean transitToGoal(Cell bin, List<FeatureFunction> featureFunctions, int sentenceLength) {
- this.sortedNodes = new ArrayList<HGNode>();
++ this.sortedNodes = new ArrayList<>();
+ HGNode goalItem = null;
+
+ for (HGNode antNode : bin.getSortedNodes()) {
+ if (antNode.lhs == this.goalSymbol) {
+ float logP = antNode.bestHyperedge.getBestDerivationScore();
- List<HGNode> antNodes = new ArrayList<HGNode>();
++ List<HGNode> antNodes = new ArrayList<>();
+ antNodes.add(antNode);
+
+ float finalTransitionLogP = ComputeNodeResult.computeFinalCost(featureFunctions, antNodes,
+ 0, sentenceLength, null, this.chart.getSentence());
+
- List<HGNode> previousItems = new ArrayList<HGNode>();
++ List<HGNode> previousItems = new ArrayList<>();
+ previousItems.add(antNode);
+
+ HyperEdge dt = new HyperEdge(null, logP + finalTransitionLogP, finalTransitionLogP,
+ previousItems, null);
+
+ if (null == goalItem) {
+ goalItem = new HGNode(0, sentenceLength + 1, this.goalSymbol, null, dt, logP
+ + finalTransitionLogP);
+ this.sortedNodes.add(goalItem);
+ } else {
+ goalItem.addHyperedgeInNode(dt);
+ }
+ } // End if item.lhs == this.goalSymID
+ } // End foreach Item in bin.get_sorted_items()
+
+ int itemsInGoalBin = getSortedNodes().size();
+ if (1 != itemsInGoalBin) {
+ LOG.error("the goal_bin does not have exactly one item");
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * a note about pruning: when a hyperedge gets created, it first needs to pass through
+ * shouldPruneEdge filter. Then, if it does not trigger a new node (i.e. will be merged to an old
+ * node), then does not trigger pruningNodes. If it does trigger a new node (either because its
+ * signature is new or because its logP is better than the old node's logP), then it will trigger
+ * pruningNodes, which might causes *other* nodes got pruned as well
+ * */
+
+ /**
+ * Creates a new hyperedge and adds it to the chart, subject to pruning. The logic of this
+ * function is as follows: if the pruner permits the edge to be added, we build the new edge,
+ * which ends in an HGNode. If this is the first time we've built an HGNode for this point in the
+ * graph, it gets added automatically. Otherwise, we add the hyperedge to the existing HGNode,
+ * possibly updating the HGNode's cache of the best incoming hyperedge.
+ *
+ * @return the new hypernode, or null if the cell was pruned.
+ */
+ HGNode addHyperEdgeInCell(ComputeNodeResult result, Rule rule, int i, int j, List<HGNode> ants,
+ SourcePath srcPath, boolean noPrune) {
+
+// System.err.println(String.format("ADD_EDGE(%d-%d): %s", i, j, rule.getRuleString()));
+// if (ants != null) {
+// for (int xi = 0; xi < ants.size(); xi++) {
+// System.err.println(String.format(" -> TAIL %s", ants.get(xi)));
+// }
+// }
+
+ List<DPState> dpStates = result.getDPStates();
+ float pruningEstimate = result.getPruningEstimate();
+ float transitionLogP = result.getTransitionCost();
+ float finalizedTotalLogP = result.getViterbiCost();
+
+ /**
+ * Here, the edge has passed pre-pruning. The edge will be added to the chart in one of three
+ * ways:
+ *
+ * 1. If there is no existing node, a new one gets created and the edge is its only incoming
+ * hyperedge.
+ *
+ * 2. If there is an existing node, the edge will be added to its list of incoming hyperedges,
+ * possibly taking place as the best incoming hyperedge for that node.
+ */
+
+ HyperEdge hyperEdge = new HyperEdge(rule, finalizedTotalLogP, transitionLogP, ants, srcPath);
+ HGNode newNode = new HGNode(i, j, rule.getLHS(), dpStates, hyperEdge, pruningEstimate);
+
+ /**
+ * each node has a list of hyperedges, need to check whether the node is already exist, if
+ * yes, just add the hyperedges, this may change the best logP of the node
+ * */
+ HGNode oldNode = this.nodesSigTbl.get(newNode.signature());
+ if (null != oldNode) { // have an item with same states, combine items
+ this.chart.nMerged++;
+
+ /**
+ * the position of oldItem in this.heapItems may change, basically, we should remove the
+ * oldItem, and re-insert it (linear time), this is too expense)
+ **/
+ if (newNode.getScore() > oldNode.getScore()) { // merge old to new: semiring plus
+
+ newNode.addHyperedgesInNode(oldNode.hyperedges);
+ // This will update the HashMap, so that the oldNode is destroyed.
+ addNewNode(newNode);
+ } else {// merge new to old, does not trigger pruningItems
+ oldNode.addHyperedgesInNode(newNode.hyperedges);
+ }
+
+ } else { // first time item
+ this.chart.nAdded++; // however, this item may not be used in the future due to pruning in
+ // the hyper-graph
+ addNewNode(newNode);
+ }
+
+ return newNode;
+ }
+
+ List<HGNode> getSortedNodes() {
+ ensureSorted();
+ return this.sortedNodes;
+ }
+
+ Map<Integer, SuperNode> getSortedSuperItems() {
+ ensureSorted();
+ return this.superNodesTbl;
+ }
+
+ // ===============================================================
+ // Private Methods
+ // ===============================================================
+
+ /**
+ * two cases this function gets called (1) a new hyperedge leads to a non-existing node signature
+ * (2) a new hyperedge's signature matches an old node's signature, but the best-logp of old node
+ * is worse than the new hyperedge's logP
+ * */
+ private void addNewNode(HGNode node) {
+ this.nodesSigTbl.put(node.signature(), node); // add/replace the item
+ this.sortedNodes = null; // reset the list
+
+// System.err.println(String.format("** NEW NODE %s %d %d", Vocabulary.word(node.lhs), node.i, node.j));
+
+ // since this.sortedItems == null, this is not necessary because we will always call
+ // ensure_sorted to reconstruct the this.tableSuperItems
+ // add a super-items if necessary
+ SuperNode si = this.superNodesTbl.get(node.lhs);
+ if (null == si) {
+ si = new SuperNode(node.lhs);
+ this.superNodesTbl.put(node.lhs, si);
+ }
+ si.nodes.add(node);// TODO what about the dead items?
+ }
+
+ /**
+ * get a sorted list of Nodes in the cell, and also make sure the list of node in any SuperItem is
+ * sorted, this will be called only necessary, which means that the list is not always sorted,
+ * mainly needed for goal_bin and cube-pruning
+ */
+ private void ensureSorted() {
+ if (null == this.sortedNodes) {
+
+ // get sortedNodes.
+ this.sortedNodes = new ArrayList<>(this.nodesSigTbl.size());
+ for (HGNode node : this.nodesSigTbl.values()) {
+ this.sortedNodes.add(node);
+ }
+
+ // sort the node in an decreasing-LogP order
+ this.sortedNodes.sort(HGNode.inverseLogPComparator);
+
+ // TODO: we cannot create new SuperItem here because the DotItem link to them.
+ // Thus, we clear nodes from existing SuperNodes
+ for (SuperNode superNode : this.superNodesTbl.values()) {
+ superNode.nodes.clear();
+ }
+
+ for (HGNode node : this.sortedNodes) {
+ SuperNode superNode = this.superNodesTbl.get(node.lhs);
+ checkNotNull(superNode, "Does not have super Item, have to exist");
+ superNode.nodes.add(node);
+ }
+
+ // Remove SuperNodes who may not contain any nodes anymore due to pruning
+ for (Iterator<Entry<Integer, SuperNode>> it = this.superNodesTbl.entrySet().iterator(); it.hasNext(); ) {
+ Entry<Integer, SuperNode> entry = it.next();
+ if (entry.getValue().nodes.isEmpty()) {
+ it.remove();
+ }
+ }
+ }
+ }
+}